From d224d1c19ab82ff29730777fc2f98fec1d60276f Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Mon, 5 Oct 2026 03:55:25 -0700 Subject: [PATCH 01/99] fix(wisp): --age 0s treats every wisp as abandoned, even one Dolt rounded into the future (#7236) findAbandonedWisps compared now.Sub(issue.UpdatedAt) > ageThreshold unconditionally. Dolt's DATETIME(0) column rounds fractional seconds half-up on write rather than truncating: a wisp created in the second half of a wall-clock second can read back with updated_at up to 0.5s in the future, making the comparison negative and failing the check for a wisp that is, by definition, not yet one second old. --age 0s's own documented contract ("hasn't been updated in --age duration") means every wisp qualifies trivially -- nothing has been updated in the last zero seconds. Extracted the comparison into wispIsAbandoned(now, updatedAt, ageThreshold), which short-circuits true whenever ageThreshold <= 0, bypassing the clock comparison (and its rounding exposure) entirely for that case. New TestWispIsAbandoned is a pure, deterministic unit test -- the original TestCLI_WispGCExcludeType needs a live Dolt test container and the reporter's own 60 real runs against one could not reliably reproduce the race, so a timing-dependent regression test would be exactly as unreliable. Confirmed red against the old inline comparison (the two age-0s cases failed) before extracting the fix, green after. go build/vet/gofmt clean. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0185qSpS74AbkmHoKRPf2yDD --- CHANGELOG.md | 8 +++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/wisp.go | 21 ++++++++++- cmd/bd/wisp_abandoned_test.go | 68 +++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/wisp_abandoned_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4cb8a05..986e0cb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -433,6 +433,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 role through (see *Added*). A missing id is a plain not-found after one `getIssue`, instead of a partial-id search refusal, and `bd update --claim` answers a not-found gc classifies. +- **`bd mol wisp gc --age 0s` can miss a just-created wisp.** Dolt's + `DATETIME(0)` column rounds fractional seconds half-up on write rather + than truncating, so a wisp created in the second half of a wall-clock + second can read back with `updated_at` up to 0.5s in the future. The GC's + age check (`now.Sub(updatedAt) > ageThreshold`) then compares against a + negative duration and fails, even though `--age 0s`'s own contract is + that every wisp qualifies. `ageThreshold <= 0` now bypasses the clock + comparison entirely (#7236). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 67f4cb9ed..854c802df 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1266,6 +1266,7 @@ go_test( "where_cgo_test.go", "where_embedded_test.go", "where_test.go", + "wisp_abandoned_test.go", "wisp_gc_active_embedded_test.go", "wisp_gc_purge_closed_embedded_test.go", "wisp_proxied_integration_test.go", diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index a1b7ac27e..9326cdb00 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -822,6 +822,25 @@ func runWispGC(cmd *cobra.Command, args []string) error { return nil } +// wispIsAbandoned reports whether a wisp counts as abandoned for GC +// purposes: it has not been updated within ageThreshold of now. +// +// ageThreshold <= 0 (--age 0s and below) always reports true, bypassing +// the clock comparison entirely — this matches the GC's own documented +// contract ("hasn't been updated in --age duration"): nothing has been +// updated in the last zero seconds. It also matters for a reason the +// comparison alone does not handle: Dolt's DATETIME(0) column rounds +// fractional seconds half-up on write rather than truncating (#7236), so +// a wisp created in the second half of a wall-clock second can read back +// with updatedAt up to 0.5s in the future. now.Sub(updatedAt) is then +// negative, failing a strict > comparison for a wisp that is, by +// definition, not yet one second old — exactly what --age 0s's "every +// wisp qualifies" contract exists to avoid depending on in the first +// place. +func wispIsAbandoned(now, updatedAt time.Time, ageThreshold time.Duration) bool { + return ageThreshold <= 0 || now.Sub(updatedAt) > ageThreshold +} + func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThreshold time.Duration, excludeTypes []types.IssueType) ([]*types.Issue, error) { ephemeralFlag := true filter := types.IssueFilter{ @@ -860,7 +879,7 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre if isProtectedWisp(issue, blockedSet, protectedStatuses) { continue } - if now.Sub(issue.UpdatedAt) > ageThreshold { + if wispIsAbandoned(now, issue.UpdatedAt, ageThreshold) { abandoned = append(abandoned, issue) } } diff --git a/cmd/bd/wisp_abandoned_test.go b/cmd/bd/wisp_abandoned_test.go new file mode 100644 index 000000000..b3e5e4bbf --- /dev/null +++ b/cmd/bd/wisp_abandoned_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "testing" + "time" +) + +// TestWispIsAbandoned pins the #7236 fix: --age 0s (ageThreshold <= 0) must +// treat every wisp as abandoned regardless of its updatedAt, including a +// wisp whose stored updatedAt reads back AFTER now — the shape Dolt's +// DATETIME(0) half-up rounding on write produces for a wisp created in the +// second half of a wall-clock second (#7236). Before this fix, that case +// failed the strict now.Sub(updatedAt) > ageThreshold comparison because +// the difference is negative. +func TestWispIsAbandoned(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + + cases := []struct { + name string + updatedAt time.Time + age time.Duration + want bool + }{ + { + name: "age 0s: a wisp updated exactly now is abandoned", + updatedAt: now, + age: 0, + want: true, + }, + { + name: "age 0s: a wisp whose stored updatedAt rounded into the future is still abandoned", + updatedAt: now.Add(500 * time.Millisecond), + age: 0, + want: true, + }, + { + name: "a negative age behaves like zero", + updatedAt: now.Add(500 * time.Millisecond), + age: -time.Second, + want: true, + }, + { + name: "positive age: a fresh wisp is not abandoned", + updatedAt: now, + age: time.Hour, + want: false, + }, + { + name: "positive age: a wisp past the threshold is abandoned", + updatedAt: now.Add(-2 * time.Hour), + age: time.Hour, + want: true, + }, + { + name: "positive age: exactly at the threshold is not abandoned (strict >, unchanged)", + updatedAt: now.Add(-time.Hour), + age: time.Hour, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wispIsAbandoned(now, tc.updatedAt, tc.age); got != tc.want { + t.Errorf("wispIsAbandoned(%v, %v, %v) = %v, want %v", now, tc.updatedAt, tc.age, got, tc.want) + } + }) + } +} From 9f3f5695536db26ac6fc0eed958ef6173c2a624f Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Fri, 9 Oct 2026 19:04:38 -0700 Subject: [PATCH 02/99] chore: empty commit to retrigger mergeability recompute GitHub's mergeStateStatus has shown DIRTY/CONFLICTING since 2026-10-06, but a real `git merge --no-commit --no-ff` against current upstream/main auto-merges cleanly on all 3 touched files -- no actual conflict. This commit forces GitHub to recompute. Co-Authored-By: Claude Sonnet 5 From 2735bc5a1366a51841221f7bfc058fb8a4093d7d Mon Sep 17 00:00:00 2001 From: Eddie the Engineer Date: Sun, 11 Oct 2026 04:30:14 +0000 Subject: [PATCH 03/99] test(wisp): pin the gc call site and drop the 1ms age from the embedded test Folds the tests from #7472 into the #7236 fix. TestWispIsAbandoned pins the predicate. It cannot see whether findAbandonedWisps still decides age through it. TestFindAbandonedWisps AgesAtStoragePrecision drives findAbandonedWisps through a new wispGCNow clock seam with a wisp stored rounded into the future (written 12:00:00.7, stored 12:00:01): - --age 0s at 12:00:00.9 reclaims it. This fails against the old inline now.Sub(updated_at) > threshold. - --age 1h leaves it alone at that instant and reclaims it two hours later. - --age 1s at 12:00:01.5 leaves it alone: a positive threshold does not credit idle time that has not passed. TestWispGCProtectsActiveWisps ran gc with --age 1ms and relied on creation order to keep the most recently touched reclaimable wisp out of the rounding window. --age 0s states the intent ("every wisp is old enough") and is immune to the half-second rounding, so the ordering caveat goes. Co-authored-by: Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/bd/wisp.go | 7 ++- cmd/bd/wisp_abandoned_test.go | 70 ++++++++++++++++++++++++++ cmd/bd/wisp_gc_active_embedded_test.go | 18 +++---- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index 9326cdb00..82d0ee530 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -841,6 +841,11 @@ func wispIsAbandoned(now, updatedAt time.Time, ageThreshold time.Duration) bool return ageThreshold <= 0 || now.Sub(updatedAt) > ageThreshold } +// wispGCNow is the clock seam for findAbandonedWisps' age check, swappable in +// tests. Both gc routes read it, so a test that swaps it must not call +// t.Parallel(). +var wispGCNow = time.Now + func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThreshold time.Duration, excludeTypes []types.IssueType) ([]*types.Issue, error) { ephemeralFlag := true filter := types.IssueFilter{ @@ -867,7 +872,7 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre return nil, err } - now := time.Now() + now := wispGCNow() var abandoned []*types.Issue for _, issue := range issues { if r.IsInfraTypeCtx(ctx, issue.IssueType) { diff --git a/cmd/bd/wisp_abandoned_test.go b/cmd/bd/wisp_abandoned_test.go index b3e5e4bbf..e18c3397a 100644 --- a/cmd/bd/wisp_abandoned_test.go +++ b/cmd/bd/wisp_abandoned_test.go @@ -1,8 +1,11 @@ package main import ( + "context" "testing" "time" + + "github.com/steveyegge/beads/internal/types" ) // TestWispIsAbandoned pins the #7236 fix: --age 0s (ageThreshold <= 0) must @@ -66,3 +69,70 @@ func TestWispIsAbandoned(t *testing.T) { }) } } + +// fixedWispsMolReader serves findAbandonedWisps a fixed set of wisps with +// nothing blocked, no custom statuses and no dependents. It embeds the +// molReader INTERFACE, so a read the age scan does not make today panics +// rather than quietly answering zero. +type fixedWispsMolReader struct { + molReader + wisps []*types.Issue +} + +func (r fixedWispsMolReader) SearchIssues(context.Context, string, types.IssueFilter) ([]*types.Issue, error) { + return r.wisps, nil +} +func (fixedWispsMolReader) GetBlockedIssues(context.Context, types.WorkFilter) ([]*types.BlockedIssue, error) { + return nil, nil +} +func (fixedWispsMolReader) GetCustomStatusesDetailed(context.Context) ([]types.CustomStatus, error) { + return nil, nil +} +func (fixedWispsMolReader) IsInfraTypeCtx(context.Context, types.IssueType) bool { return false } +func (fixedWispsMolReader) FindWispDependentsRecursive(context.Context, []string) (map[string]bool, error) { + return nil, nil +} + +// TestFindAbandonedWispsAgesAtStoragePrecision pins the call site that +// TestWispIsAbandoned cannot see: findAbandonedWisps must decide age through +// wispIsAbandoned. A wisp written at 12:00:00.7 is stored as 12:00:01, and a +// gc at 12:00:00.9 with --age 0s must still reclaim it; the raw +// now.Sub(updated_at) > threshold that #7236 replaced reads -100ms and misses +// it. A one-hour threshold must leave the same wisp alone at that instant and +// reclaim it two hours later, so dropping the age check fails too. --age 1s +// at 12:00:01.5 must leave it alone: only half a second has passed since the +// stored second, so a positive threshold never credits idle time that has +// not elapsed. +func TestFindAbandonedWispsAgesAtStoragePrecision(t *testing.T) { + base := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) + wisp := &types.Issue{ + ID: "gc-idle", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Ephemeral: true, + UpdatedAt: base.Add(700 * time.Millisecond).Round(time.Second), // 12:00:01 + } + old := wispGCNow + t.Cleanup(func() { wispGCNow = old }) + + for _, tc := range []struct { + now time.Time + threshold time.Duration + want int + }{ + {base.Add(900 * time.Millisecond), 0, 1}, + {base.Add(900 * time.Millisecond), time.Hour, 0}, + {base.Add(2 * time.Hour), time.Hour, 1}, + {base.Add(1500 * time.Millisecond), time.Second, 0}, + } { + wispGCNow = func() time.Time { return tc.now } + got, err := findAbandonedWisps(context.Background(), fixedWispsMolReader{wisps: []*types.Issue{wisp}}, false, tc.threshold, nil) + if err != nil { + t.Fatalf("findAbandonedWisps(--age %s): %v", tc.threshold, err) + } + if len(got) != tc.want { + t.Errorf("findAbandonedWisps(--age %s) at %s, updated_at 12:00:01: got %d wisps, want %d", + tc.threshold, tc.now.Format("15:04:05.000"), len(got), tc.want) + } + } +} diff --git a/cmd/bd/wisp_gc_active_embedded_test.go b/cmd/bd/wisp_gc_active_embedded_test.go index 3d62d9f93..7ab85a319 100644 --- a/cmd/bd/wisp_gc_active_embedded_test.go +++ b/cmd/bd/wisp_gc_active_embedded_test.go @@ -28,14 +28,6 @@ func TestWispGCProtectsActiveWisps(t *testing.T) { // category. "reviewing" is wip (protected); "triaging" is active, which // behaves like plain open and stays reclaimable — that pair also guards // against the predicate degenerating into "protect everything". - // - // Ordering matters: every wisp expected to be RECLAIMED is created and - // last-touched first, before the protected ones. The age predicate is - // now.Sub(updated_at) > threshold against a timestamp written by the - // database, so the most recently touched bead can briefly appear - // not-yet-stale if the DB clock runs ahead of the test process. Protected - // wisps are excluded regardless of age, so only the reclaimable - // assertions are sensitive to this. bdCommand(t, bd, dir, "config", "set", "status.custom", "reviewing:wip,triaging:active") // Genuinely abandoned: an idle open ephemeral wisp. Must be reclaimed. @@ -66,10 +58,12 @@ func TestWispGCProtectsActiveWisps(t *testing.T) { customWIP := bdCreate(t, bd, dir, "custom wip step", "--ephemeral").ID bdCommand(t, bd, dir, "update", customWIP, "--status", "reviewing") - // Every wisp above was created well over 1ms ago by the time gc runs, so a - // 1ms threshold makes them all "stale" by updated_at. Only the genuinely - // abandoned ones should be reclaimed. - out := bdCommand(t, bd, dir, "mol", "wisp", "gc", "--age", "1ms", "--dry-run", "--json") + // --age 0s makes every wisp above old enough to reclaim. Any positive --age + // can leave the most recently touched one reading as too young, because + // updated_at is stored at whole-second precision and Dolt rounds the write + // up to half a second into the future (#7236). Protection is checked + // before age, so only the genuinely abandoned ones should be reclaimed. + out := bdCommand(t, bd, dir, "mol", "wisp", "gc", "--age", "0s", "--dry-run", "--json") var res struct { CleanedIDs []string `json:"cleaned_ids"` From 225ed7f7bb53c449eebd6a1b077e9d435e9f43d0 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 11 Oct 2026 05:34:51 +0000 Subject: [PATCH 04/99] test(backend/http/corpus): served MC command corpus against the reference server profile (S23) (#7483) * test(backend/http/corpus): served command corpus against the reference server profile Add the S23 suite: every distinct bd command shape gc and its packs and formulas issue against a work store (124 shapes in testdata/corpus.yaml), run by the real bd binary against an in-process bd serve and compared with the same command on an embedded workspace. The server is bound in non-loopback mode with bearer auth. A reference proxy masks it to a deployed bd 1.3.0 server's wire surface (testdata/reference_server.json): exactly that server's capabilities and handshake members (no wire_revision), a 404 for operations it does not publish, and a 400 unknown_parameter for query parameters and request members it does not declare. Every bd run sets BEADS_ACTOR and CLAUDE_SESSION_ID. A shape passes when the served run exits like the embedded run, carries no refusal, and matches its output (json/ids/count/text) and its verify reads. Known gaps are marked `pending` with the slice that fixes them. A pending shape still runs, and it fails once it passes, so removing the mark is how a slice proves itself. Permanent refusals are listed in testdata/allowlist.yaml with reasons. A limited listing that cuts through a tie set (same priority, created_at in one second) lands on different rows on two workspaces seeded seconds apart; such a shape names its unlimited form in `prefix_of`, and each side's limited rows must be an ordered prefix of that side's own unlimited listing. //backend/http/corpus:corpus_test takes //cmd/bd:bd_for_tests (size large, timeout long), so under bazel the suite runs instead of skipping on a missing BEADS_TEST_BD_BINARY. Stacked on S6b/c/d, whose marks this file no longer carries: besides the S6b/S6c/S6d marks, seven list shapes once marked S6c+S15 (their --limit 0 list walks the keyset cursor without S15) and the S6c+S16 last-step close (an ordinary epic root never auto-closes) already pass on it. On S6b/c/d (bazel, BEADS_TEST_EMBEDDED_DOLT=1): 97 pass, 25 pending, 3 allowlisted. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): hand the corpus server the LeaseReclaimer role main gained the LeaseReclaimer role (issues.reclaim) after S23 was written, and httpapi.Config needs it when built from roles rather than a Provider, so on the restacked S6bcd TestServedCorpus could not bind its in-process server ("no database source"). The harness now passes s.LeaseReclaimer() with the rest, as backend/http's e2e harness does. The reference proxy still masks the issues.reclaim token, so gc-reclaim stays pending. On the restacked S6bcd (bazel, BEADS_TEST_EMBEDDED_DOLT=1): 97 pass, 25 pending, 3 allowed, unchanged from before the restack. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): run the corpus in CI, hold allowlisted shapes Review fixes for #7483 (review attempt 1): both majors, and four of the seven minors and nits. No CI lane ran TestServedCorpus. corpus_test sets neither BEADS_TEST_EMBEDDED_DOLT nor BEADS_HTTP_TEST_REQUIRED, so the suite skipped everywhere. corpus_served_test, an embedded-tagged sh_test over go_test_variant.sh like http_served_test, runs it with both set and with Bazel's prebuilt bd, and pr_risk_bazel_coverage_test pins its args and env. Every other file in the package builds only under cgo, so a new cgo-free test fails a BEADS_HTTP_TEST_REQUIRED=1 run that would otherwise pass having run nothing. README.md and engdocs/TESTING.md say which lane runs the suite. The allowlist branch returned before the embedded exit check, so an allowlisted shape was never held to what it says on the embedded backend. The embedded run is now checked against the shape's expect_exit and expect_stderr before the allowlist branch, and an embedded mismatch fails as a corpus bug whatever the shape's pending mark or allowlist entry. TestCorpusFileIsWellFormed requires an allowlisted shape to state expect_exit and to carry nothing the allowlist would skip (compare: exit, no verify). The three allowlisted shapes now pin their embedded refusals, and the bd sql reasons say the command needs a sql-server workspace instead of calling it local-only. Minors and nits taken: - two stale-guard shapes (--if-assignee nobody, --if-status open) expect exit 13 on both sides; each kills a client that drops that guard; - defer_until and due_at are compared as given instead of scrubbed to , which kills a client that shifts --defer; - gc-reclaim's pending mark names SRV-1 alone: the reference profile does not advertise issues.reclaim, so the client refuses at its capability preflight; - refusalMarkers matches "does not advertise capability" unquoted, which also catches the JSON-escaped preflight error. Declined: - a failure on empty-vs-empty comparisons, which needs an allow_empty opt-in. Review attempt 2 seeded rows for two of the three ready shapes that compared two empty lists (below); gc-ready-graph-anchor still does; - the context route's query check in the reference proxy: the server already answers an unknown query parameter there with a 400 unknown_parameter, which the proxy passes through, so the served run still fails; only the proxy's own refusal note is missing; - a shared server constructor for the two e2e harnesses, which the review marks optional. Review attempt 2: four of its six minors and nits. - gc-ready-ephemeral-assignee-unlimited and gc-ready-assigned-tier compared two empty lists, so a client that sent the wrong assignee or dropped --include-ephemeral passed both. Two seed rows assigned to the worker, an open task (P2) and an open wisp (P1), give them [e2e-wisp-wr e2e-wr] and [e2e-wisp-wr]. The priorities differ, so --limit=1 cuts no tie. A client that rewrites the assignee, or drops include_ephemeral, now fails both shapes; without the rows it passed them. - templatePattern looped forever on a profile path with an unclosed "{", growing its pattern without bound. It now returns an error, which loadReferenceProfile reports with the operation and path, and TestCorpusFileIsWellFormed loads the profile, so corpus_test catches a bad edit without the embedded engine. - TestCorpusFileIsWellFormed compiles every expect_stderr and every allowlist refusal, and runShape compiles expect_stderr once and fails the shape as a corpus bug instead of panicking on a bad pattern. - README.md documents the stdin, setup, ordered and ignore_fields fields. Not in this commit: - the PR description (shape counts, allowlist reasons, gc-reclaim's SRV-1 mark, the Bazel run line) is brought up to date when this commit is pushed; - hoisting runBD, minimalBDEnv, buildBD and the skip gate into internal/testutil, or cross-reference comments in their three copies, which the review marks optional and which reaches outside this package. The full corpus gives 127 shapes: 99 pass, 25 pending, 3 allowed, with no shape changing status from the attempt 1 fixup, on this tree and merged with main at 0a163ca285. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- BUILD.bazel | 4 + backend/http/corpus/BUILD.bazel | 123 +++ backend/http/corpus/README.md | 120 +++ backend/http/corpus/corpus_test.go | 817 ++++++++++++++++++ backend/http/corpus/harness_test.go | 312 +++++++ backend/http/corpus/reference_proxy_test.go | 279 ++++++ .../http/corpus/required_corpus_tier_test.go | 23 + backend/http/corpus/testdata/allowlist.yaml | 29 + backend/http/corpus/testdata/corpus.yaml | 735 ++++++++++++++++ .../corpus/testdata/reference_server.json | 595 +++++++++++++ cmd/bd/BUILD.bazel | 1 + engdocs/TESTING.md | 2 +- scripts/pr_risk_bazel_coverage_test.go | 6 + 13 files changed, 3045 insertions(+), 1 deletion(-) create mode 100644 backend/http/corpus/BUILD.bazel create mode 100644 backend/http/corpus/README.md create mode 100644 backend/http/corpus/corpus_test.go create mode 100644 backend/http/corpus/harness_test.go create mode 100644 backend/http/corpus/reference_proxy_test.go create mode 100644 backend/http/corpus/required_corpus_tier_test.go create mode 100644 backend/http/corpus/testdata/allowlist.yaml create mode 100644 backend/http/corpus/testdata/corpus.yaml create mode 100644 backend/http/corpus/testdata/reference_server.json diff --git a/BUILD.bazel b/BUILD.bazel index 104409817..7b10becbc 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -300,6 +300,7 @@ filegroup( "//backend:repo_doc_files", "//backend/conformance:repo_doc_files", "//backend/http:repo_doc_files", + "//backend/http/corpus:repo_doc_files", "//beadserrors:repo_doc_files", "//cmd/bd:repo_doc_files", "//cmd/bd/doctor:repo_doc_files", @@ -533,6 +534,7 @@ filegroup( "//backend:repo_go_srcs", "//backend/conformance:repo_go_srcs", "//backend/http:repo_go_srcs", + "//backend/http/corpus:repo_go_srcs", "//beadserrors:repo_go_srcs", "//cmd/bd:repo_go_srcs", "//cmd/bd/doctor:repo_go_srcs", @@ -751,6 +753,7 @@ filegroup( "//backend:repo_go_test_srcs", "//backend/conformance:repo_go_test_srcs", "//backend/http:repo_go_test_srcs", + "//backend/http/corpus:repo_go_test_srcs", "//beadserrors:repo_go_test_srcs", "//cmd/bd:repo_go_test_srcs", "//cmd/bd/doctor:repo_go_test_srcs", @@ -978,6 +981,7 @@ filegroup( "//backend:repo_other_files", "//backend/conformance:repo_other_files", "//backend/http:repo_other_files", + "//backend/http/corpus:repo_other_files", "//beadserrors:repo_other_files", "//cmd/bd:repo_other_files", "//cmd/bd/doctor:repo_other_files", diff --git a/backend/http/corpus/BUILD.bazel b/backend/http/corpus/BUILD.bazel new file mode 100644 index 000000000..7a6f87f43 --- /dev/null +++ b/backend/http/corpus/BUILD.bazel @@ -0,0 +1,123 @@ +load("@rules_go//go:def.bzl", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +go_test( + name = "corpus_test", + # The corpus runs every shape through the real bd (bazeltest.PrebuiltBD), + # served and embedded, under BEADS_TEST_EMBEDDED_DOLT=1 (corpus_served_test + # below); it skips otherwise. + size = "large", + timeout = "long", + srcs = [ + "corpus_test.go", + "harness_test.go", + "reference_proxy_test.go", + "required_corpus_tier_test.go", + ], + data = glob(["testdata/**"]) + ["//cmd/bd:bd_for_tests"], + env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"}, + deps = [ + "//internal/httpapi", + "//internal/httpapi/apigen", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/testutil/bazeltest", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) + +# corpus_test again, with the corpus tier on: BEADS_TEST_EMBEDDED_DOLT=1 runs +# TestServedCorpus and BEADS_HTTP_TEST_REQUIRED=1 turns a skip of it into a +# failure. corpus_test alone skips TestServedCorpus in every lane, because no +# lane sets those variables for it. It repeats corpus_test's `env` +# (go_test_variant.sh). +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "corpus_served_test", + size = "large", + # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s + # action limit (eternal under --config=embedded). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :corpus_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + ], + data = [ + ":corpus_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_HTTP_TEST_REQUIRED": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + }, + tags = ["embedded"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/backend/http/corpus/README.md b/backend/http/corpus/README.md new file mode 100644 index 000000000..57e17550b --- /dev/null +++ b/backend/http/corpus/README.md @@ -0,0 +1,120 @@ +# Served command corpus + +This suite is the working definition of "bd fully supports a remote HTTP +backend". It runs every distinct `bd` command shape that an orchestrator (gc) +and its packs and formulas issue against a work store. Each shape runs +against a served workspace and is compared with the same command on an +embedded workspace. + +## What runs + +`TestServedCorpus` does the following: + +1. Builds `bd` from this tree. `BEADS_TEST_BD_BINARY=` reuses a prebuilt + binary instead. +2. Starts an in-process `bd serve` (`httpapi.Listen` over embedded Dolt) the + way a deployed server is bound: `AllowNonLoopback` plus a bearer token + file. The socket is still `127.0.0.1`, but the bind mode decides every + behaviour the server keys on, including its refusal of unlimited reads. +3. Puts a reference proxy in front of the server. The proxy makes it answer + like the deployed reference server described in + `testdata/reference_server.json`: + - the context handshake publishes exactly that server's capability set and + its `ContextResponse` members (so no `wire_revision`); + - an operation the reference server does not publish gets the server's own + 404; + - a query parameter or top-level request member it does not declare gets + the server's own 400 `unknown_parameter`. +4. Runs `bd connect` on one workspace and `bd init` on another. Both get the + `seed` from `testdata/corpus.yaml`: one graph of named rows, plus bulk + graphs that put more than one server page of ready rows and wisps in the + store. Both sides use `BEADS_ACTOR=worker` and set `CLAUDE_SESSION_ID`. +5. Runs each shape as a subtest, on both sides. The embedded run must exit + with the shape's `expect_exit` and match its `expect_stderr`. If it does + not, the shape is wrong or bd changed, and the subtest fails as a corpus + bug that no pending mark or allowlist entry hides. The served run must: + - exit like the embedded run; + - carry no refusal (`ErrUnsupported`, a v0-wire refusal, the + loopback-only 400, or a reference-profile 404/400); + - match the embedded output under the shape's `compare` mode, after + timestamps and revision tokens are dropped (but not `defer_until` or + `due_at`: a shape sets those itself, as absolute dates, and they must + match exactly); + - for writes, give the same JSON from every `verify` read on both sides + afterwards. + +The run takes about five minutes. + +``` +CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ + go test -tags gms_pure_go -run 'TestServedCorpus|TestCorpusFile' ./backend/http/corpus/ +``` + +In CI, `//backend/http/corpus:corpus_served_test` runs the suite with both +variables set, in the Embedded Dolt lane (`--config=embedded`; see +[engdocs/TESTING.md](../../../engdocs/TESTING.md)). `corpus_test` on its own +sets neither, so it skips `TestServedCorpus`. + +Two environment variables change the run: +- `BEADS_CORPUS_REPORT=` writes a JSON summary with per-shape status + and request counts. +- `BEADS_CORPUS_SKIP_PENDING=1` skips pending shapes without running them. + +## The data + +| File | Holds | +|---|---| +| `testdata/corpus.yaml` | assets, the seed, and the shapes | +| `testdata/allowlist.yaml` | permanent refusals, each with its reason and the refusal text the served run must match | +| `testdata/reference_server.json` | the reference server's wire surface | + +Each shape in `corpus.yaml` has these fields: + +| Field | Meaning | +|---|---| +| `id` | `gc-*` for orchestrator subprocess shapes, `mc-*` for pack/formula/prompt shapes | +| `source` | where the shape comes from | +| `args` | the argv; `{file:NAME}` expands to an asset path and `{rev:ID}` to that side's current revision of `ID` | +| `stdin` | text piped to `bd` on both sides | +| `setup` | commands run on both sides before the shape; one that fails fails the shape | +| `compare` | `json` (default), `ids`, `count`, `text` or `exit` | +| `ordered` | under `json` and `ids`, array order is significant; by default arrays are sorted before comparing | +| `prefix_of` | for a limited `ids` listing: its unlimited form. Each side's limited rows must be a prefix, in order, of that side's own unlimited listing, both sides return as many rows, and the two unlimited listings hold the same ids. A limit that cuts through a tie set (same priority, `created_at` in the same second) lands on different rows on two workspaces seeded seconds apart, so the limited rows are not compared across sides directly | +| `verify` | reads run after the shape | +| `ignore_fields` | JSON keys dropped, at any depth, before a `json` comparison and before `verify` output is compared; for example comment ids, which each workspace mints itself | +| `expect_exit`, `expect_stderr` | expected outcome on both sides (exit 0 by default). An allowlisted shape must state `expect_exit`; only its embedded run is held to these | +| `pending` | a known failure | + +`pending: {slice, reason}` marks a known failure and names the slice or +slices that fix it, for example `S6c+S15`. A pending shape **still runs**: +- while it fails, the subtest is skipped and the skip records what was + observed; +- once it passes, the subtest fails with "now passes; remove the pending + mark". + +So removing a mark is how a slice proves itself, and any other failure is +unexpected and fails the suite. + +## Updating the reference profile + +`reference_server.json` was derived from two things: +- the deployed server's live `GET /v0/beads/context` capabilities; +- its published OpenAPI document (bd 1.3.0, no `wire_revision`). For each + operation the profile keeps the method, the path template, the query + parameter names, and the top-level request members. + +When the deployed server is upgraded (SRV-1), regenerate the file the same +way. Then remove the `SRV-1` pending marks that now pass. + +## Known fidelity limits + +- Responses pass through unfiltered. A response member the reference server + would not send still reaches the client. +- Only top-level request members are checked; nested members (for example + `batchApply` items) are not. +- The served backend underneath is embedded Dolt, not the reference + deployment's storage backend. +- Out of scope, because they are local by design or never touch a remote + store: `init`, `bootstrap`, `migrate`, `doctor`, `flatten`, `gc`, and the + `dolt` subcommands. `context`, `version`, `backup status` and `prime` + are in the corpus, but only checked for exit codes. diff --git a/backend/http/corpus/corpus_test.go b/backend/http/corpus/corpus_test.go new file mode 100644 index 000000000..96a3b095e --- /dev/null +++ b/backend/http/corpus/corpus_test.go @@ -0,0 +1,817 @@ +//go:build cgo + +// Package corpus_test is the served command-corpus suite: every distinct bd +// command shape an orchestrator (gc) and its packs/formulas issue, run by the +// real bd binary against an in-process bd serve masked to a reference +// deployment's wire surface, and compared with the same command on an embedded +// workspace. See README.md. +package corpus_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// corpusFile is testdata/corpus.yaml. +type corpusFile struct { + Files []corpusAsset `yaml:"files"` + Seed [][]string `yaml:"seed"` + Shapes []shape `yaml:"shapes"` +} + +// corpusAsset is a file the shapes reference as {file:NAME}. Exactly one of +// Content and Bulk is set. +type corpusAsset struct { + Name string `yaml:"name"` + Content string `yaml:"content"` + Bulk *bulkGraph `yaml:"bulk"` +} + +// bulkGraph generates a `bd create --graph` plan of Count independent tasks +// with ids IDPrefix-001.. so the suite can hold more rows than one server page +// without a 100-line literal in the corpus. +type bulkGraph struct { + IDPrefix string `yaml:"id_prefix"` + Count int `yaml:"count"` + Priority int `yaml:"priority"` + Ephemeral bool `yaml:"ephemeral"` + Metadata map[string]string `yaml:"metadata"` +} + +type shape struct { + ID string `yaml:"id"` + Source string `yaml:"source"` + // Args is the bd argv. Tokens may carry {file:NAME} (the side's copy of a + // corpus asset) and {rev:ID} (that side's current revision of ID). + Args []string `yaml:"args"` + Stdin string `yaml:"stdin"` + Setup [][]string `yaml:"setup"` + // Compare is how the served stdout is held to the embedded stdout: + // json (default), ids, count, text, or exit (exit code only). + Compare string `yaml:"compare"` + // Ordered keeps array order significant under json/ids. + Ordered bool `yaml:"ordered"` + // PrefixOf is the unlimited form of a limited listing (compare: ids). A + // limit cut through a tie set (same priority, created_at within one + // second) lands at a different row on two workspaces seeded seconds + // apart, so the limited rows are not compared across sides. Instead each + // side's limited rows must be, in order, a prefix of that SAME side's + // PrefixOf listing, both sides must return as many rows, and the two + // PrefixOf listings must hold the same ids. + PrefixOf []string `yaml:"prefix_of"` + // IgnoreFields are extra JSON keys dropped before comparing. + IgnoreFields []string `yaml:"ignore_fields"` + // ExpectExit is the exit code both sides must produce (default 0). An + // allowlisted shape must state it: the embedded run is held to it even + // though the served run is only held to the allowlisted refusal. + ExpectExit *int `yaml:"expect_exit"` + // ExpectStderr, when set, must match both sides' stderr (only the + // embedded side's on an allowlisted shape). + ExpectStderr string `yaml:"expect_stderr"` + // Verify are reads run on both sides after the shape and compared as json: + // how a write is checked for having done the same thing on both backends. + Verify [][]string `yaml:"verify"` + // Pending marks a known failure and names the slice that fixes it. + Pending *pending `yaml:"pending"` +} + +// wantExit is the exit code the embedded run must produce. +func (s shape) wantExit() int { + if s.ExpectExit == nil { + return 0 + } + return *s.ExpectExit +} + +type pending struct { + Slice string `yaml:"slice"` + Reason string `yaml:"reason"` +} + +// allowEntry is testdata/allowlist.yaml: a shape that is EXPECTED to be +// refused over http, permanently, for a stated reason. The served run must +// fail and match Refusal; if it ever succeeds the entry is stale and fails. +// The embedded run is still held to the shape's expect_exit and +// expect_stderr, and nothing else of the shape is checked. +type allowEntry struct { + ID string `yaml:"id"` + Reason string `yaml:"reason"` + Refusal string `yaml:"refusal"` +} + +// refusalMarkers are the texts a refusal of an operation over http carries: +// issueops.ErrUnsupported, encode.RefusedError, the client's capability +// preflight (wire.CapabilityError, which never reaches the proxy), the +// server's loopback-only unlimited-read 400, an unrouted 404, and a strict +// decoder's unknown member. +var refusalMarkers = regexp.MustCompile(`(?i)not supported by the|not expressible on the v0 wire|loopback-only|capability "[^"]+" not advertised|does not advertise capability|no such route on this server|unknown_parameter|unknown query parameter|json: unknown field`) + +// volatileKey names JSON members that legitimately differ between two +// workspaces fed the same commands: timestamps and revision tokens. +var volatileKey = regexp.MustCompile(`(^|_)at$|^revision$|^row_version$|^expected_revision$|^current_revision$`) + +// callerTimeKey names the timestamps a shape's own arguments set (bd create +// --defer, --due). They are compared as given, neither dropped as volatile +// nor scrubbed to , so a client that sends the wrong one fails; a shape +// must give them as absolute dates, since a relative one resolves against +// each side's own clock. +var callerTimeKey = map[string]bool{"defer_until": true, "due_at": true} + +var timestampText = regexp.MustCompile(`\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?`) + +func loadCorpus(t *testing.T) (*corpusFile, map[string]allowEntry) { + t.Helper() + raw, err := os.ReadFile("testdata/corpus.yaml") + if err != nil { + t.Fatalf("read corpus: %v", err) + } + var c corpusFile + dec := yaml.NewDecoder(strings.NewReader(string(raw))) + dec.KnownFields(true) + if err := dec.Decode(&c); err != nil { + t.Fatalf("parse testdata/corpus.yaml: %v", err) + } + rawAllow, err := os.ReadFile("testdata/allowlist.yaml") + if err != nil { + t.Fatalf("read allowlist: %v", err) + } + var entries []allowEntry + dec = yaml.NewDecoder(strings.NewReader(string(rawAllow))) + dec.KnownFields(true) + if err := dec.Decode(&entries); err != nil { + t.Fatalf("parse testdata/allowlist.yaml: %v", err) + } + allow := map[string]allowEntry{} + for _, e := range entries { + if e.ID == "" || strings.TrimSpace(e.Reason) == "" || e.Refusal == "" { + t.Fatalf("allowlist entry %+v needs id, reason and refusal", e) + } + allow[e.ID] = e + } + return &c, allow +} + +// TestCorpusFileIsWellFormed runs without the embedded engine: it keeps the +// data honest (unique ids, a source and a slice for every pending mark, no +// allowlist entry for a shape that does not exist, no shape both pending and +// allowlisted, an allowlisted shape states its embedded exit code and +// carries no check that the allowlist would skip, every expect_stderr and +// refusal compiles, and the reference profile loads). +func TestCorpusFileIsWellFormed(t *testing.T) { + c, allow := loadCorpus(t) + seen := map[string]bool{} + for _, s := range c.Shapes { + if s.ID == "" || len(s.Args) == 0 { + t.Errorf("shape %+v needs an id and args", s) + continue + } + if seen[s.ID] { + t.Errorf("duplicate shape id %q", s.ID) + } + seen[s.ID] = true + if strings.TrimSpace(s.Source) == "" { + t.Errorf("shape %s names no source", s.ID) + } + if s.Pending != nil && (s.Pending.Slice == "" || strings.TrimSpace(s.Pending.Reason) == "") { + t.Errorf("shape %s is pending without a slice and a reason", s.ID) + } + if _, ok := allow[s.ID]; ok { + if s.Pending != nil { + t.Errorf("shape %s is both pending and allowlisted; pick one", s.ID) + } + if s.ExpectExit == nil { + t.Errorf("shape %s is allowlisted without expect_exit; state the exit code the embedded run must produce", s.ID) + } + if modeOf(s) != "exit" || len(s.Verify) > 0 { + t.Errorf("shape %s is allowlisted, so only its exit code and stderr are checked; give it compare: exit and no verify", s.ID) + } + } + switch s.Compare { + case "", "json", "ids", "count", "text", "exit": + default: + t.Errorf("shape %s: unknown compare mode %q", s.ID, s.Compare) + } + if len(s.PrefixOf) > 0 && s.Compare != "ids" { + t.Errorf("shape %s: prefix_of needs compare: ids", s.ID) + } + if _, err := regexp.Compile(s.ExpectStderr); err != nil { + t.Errorf("shape %s: expect_stderr: %v", s.ID, err) + } + } + for id, entry := range allow { + if !seen[id] { + t.Errorf("allowlist names %q, which is not a corpus shape", id) + } + if _, err := regexp.Compile(entry.Refusal); err != nil { + t.Errorf("allowlist entry %s: refusal: %v", id, err) + } + } + for _, a := range c.Files { + if (a.Content == "") == (a.Bulk == nil) { + t.Errorf("asset %s needs exactly one of content and bulk", a.Name) + } + } + loadReferenceProfile(t) +} + +// side is one workspace the corpus runs in. +type side struct { + name string + dir string + files string + env []string +} + +type outcome struct { + id, status, detail string + requests int64 +} + +// TestServedCorpus is the suite. Each shape is a subtest; a pending shape is +// still RUN and then skipped with what it did, so a slice that fixes it sees +// the subtest fail with "now passes; remove the pending mark". +// BEADS_CORPUS_SKIP_PENDING=1 skips pending shapes without running them. +// BEADS_CORPUS_REPORT= writes a JSON summary. +func TestServedCorpus(t *testing.T) { + skipUnlessEmbeddedDolt(t) + c, allow := loadCorpus(t) + profile := loadReferenceProfile(t) + bin := buildBD(t) + + serverAddr := startCorpusServer(t) + proxy := newReferenceProxy(t, profile, serverAddr) + proxyAddr := startProxy(t, proxy) + + common := []string{ + "BEADS_ACTOR=worker", + "CLAUDE_SESSION_ID=corpus-session-s23", + "BD_BACKUP_ENABLED=false", + "BD_EXPORT_AUTO=false", + } + served := &side{name: "served", dir: t.TempDir(), files: t.TempDir(), + env: append([]string{"BEADS_HTTP_TOKEN=" + proxyAddr + "=" + corpusToken}, common...)} + local := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir(), env: common} + + if r := runBD(t, bin, served.dir, served.env, "", "connect", "http://"+proxyAddr, "--expect-project-id", corpusProjectID, "--json"); r.code != 0 { + t.Fatalf("bd connect: %s", r) + } + if r := runBD(t, bin, local.dir, local.env, "", "init", "-p", corpusPrefix, "--non-interactive", "--skip-hooks", "--skip-agents", "-q"); r.code != 0 { + t.Fatalf("bd init (embedded): %s", r) + } + for _, sd := range []*side{served, local} { + writeAssets(t, c.Files, sd.files) + } + for _, args := range c.Seed { + for _, sd := range []*side{served, local} { + argv := expand(t, bin, sd, args) + if r := runBD(t, bin, sd.dir, sd.env, "", argv...); r.code != 0 { + t.Fatalf("seed %v on %s failed; the seed must use shapes that work on the reference profile: %s", args, sd.name, r) + } + } + } + proxy.takeRefusals() + + skipPending := os.Getenv("BEADS_CORPUS_SKIP_PENDING") == "1" + var outcomes []outcome + for _, s := range c.Shapes { + s := s + t.Run(s.ID, func(t *testing.T) { + o := outcome{id: s.ID} + defer func() { outcomes = append(outcomes, o) }() + if s.Pending != nil && skipPending { + o.status = "pending:" + s.Pending.Slice + t.Skipf("pending %s: %s", s.Pending.Slice, s.Pending.Reason) + } + before := proxy.requests.Load() + failures := runShape(t, bin, s, served, local, allow, proxy) + o.requests = proxy.requests.Load() - before + entry, allowed := allow[s.ID] + switch { + case hasCorpusBug(failures): + // The embedded side itself disagrees with the corpus: no pending + // mark or allowlist entry may hide that, or a pending shape that + // can never pass would sit in the corpus forever. + o.status = "FAIL(corpus)" + o.detail = failures[0] + for _, f := range failures { + t.Error(f) + } + case s.Pending != nil && len(failures) == 0: + o.status = "FAIL(pending-now-passes)" + t.Errorf("shape is marked pending %s but now passes on the reference profile; remove the pending mark from testdata/corpus.yaml", s.Pending.Slice) + case s.Pending != nil: + o.status = "pending:" + s.Pending.Slice + o.detail = failures[0] + t.Skipf("pending %s: %s\nobserved:\n%s", s.Pending.Slice, s.Pending.Reason, strings.Join(failures, "\n")) + case len(failures) > 0: + o.status = "FAIL" + o.detail = failures[0] + for _, f := range failures { + t.Error(f) + } + case allowed: + o.status = "allowed" + o.detail = entry.Reason + default: + o.status = "pass" + } + }) + } + + summarize(t, outcomes) +} + +// runShape runs one shape on both sides and returns every way the served run +// fell short. It never calls t.Error itself, so the caller can turn a pending +// shape's failures into a skip. +func runShape(t *testing.T, bin string, s shape, served, local *side, allow map[string]allowEntry, proxy *referenceProxy) []string { + t.Helper() + var fails []string + failf := func(format string, a ...any) { fails = append(fails, fmt.Sprintf(format, a...)) } + wantStderr, err := regexp.Compile(s.ExpectStderr) + if err != nil { + failf("CORPUS BUG: expect_stderr %q: %v", s.ExpectStderr, err) + return fails + } + + for _, args := range s.Setup { + for _, sd := range []*side{served, local} { + if r := runBD(t, bin, sd.dir, sd.env, "", expand(t, bin, sd, args)...); r.code != 0 { + failf("setup %v failed on %s: %s", args, sd.name, r) + return fails + } + } + } + proxy.takeRefusals() + sr := runBD(t, bin, served.dir, served.env, s.Stdin, expand(t, bin, served, s.Args)...) + wireRefusals := proxy.takeRefusals() + lr := runBD(t, bin, local.dir, local.env, s.Stdin, expand(t, bin, local, s.Args)...) + t.Logf("bd %s\nserved: %s\nembedded: exit=%d", strings.Join(s.Args, " "), sr, lr.code) + if len(wireRefusals) > 0 { + t.Logf("reference-profile refusals on the wire: %v", wireRefusals) + } + + // The embedded run is the oracle for every shape, an allowlisted one + // included: when it no longer does what the shape says, the shape is + // wrong or bd changed, whatever the served run did. + if lr.code != s.wantExit() { + failf("CORPUS BUG: the embedded run exited %d, want %d; the shape itself is wrong or bd changed: %s", lr.code, s.wantExit(), lr) + return fails + } + if s.ExpectStderr != "" && !wantStderr.MatchString(lr.stderr) { + failf("CORPUS BUG: embedded stderr does not match %q: %s", s.ExpectStderr, lr.stderr) + return fails + } + + if entry, ok := allow[s.ID]; ok { + re, err := regexp.Compile(entry.Refusal) + if err != nil { + failf("allowlist refusal regexp for %s: %v", s.ID, err) + return fails + } + if sr.code == 0 { + failf("allowlisted (%s) but the served run SUCCEEDED; drop it from testdata/allowlist.yaml and give it a comparison", entry.Reason) + } else if !re.MatchString(sr.stderr + sr.stdout) { + failf("allowlisted refusal did not match %q (a different failure than the one allowed): %s", entry.Refusal, sr) + } + return fails + } + + if m := refusalMarkers.FindString(sr.stderr + sr.stdout); m != "" { + failf("served run carries a refusal (%q): %s", m, sr) + } else if len(wireRefusals) > 0 && sr.code != 0 { + failf("served run failed on a reference-profile refusal %v: %s", wireRefusals, sr) + } + if sr.code != lr.code { + failf("exit code: served %d, embedded %d: %s", sr.code, lr.code, sr) + } + if s.ExpectStderr != "" && !wantStderr.MatchString(sr.stderr) { + failf("served stderr does not match %q (embedded does): %s", s.ExpectStderr, sr) + } + if len(fails) > 0 { + return fails + } + if len(s.PrefixOf) > 0 { + vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, s.PrefixOf)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, s.PrefixOf)...) + if vs.code != 0 || vl.code != 0 { + failf("prefix_of %v: exit served %d, embedded %d: served %s", s.PrefixOf, vs.code, vl.code, vs) + } else if d := comparePrefix(sr.stdout, lr.stdout, vs.stdout, vl.stdout); d != "" { + failf("output parity (ids, prefix_of): %s", d) + } + } else if d := compareOutputs(s, sr.stdout, lr.stdout); d != "" { + failf("output parity (%s): %s", modeOf(s), d) + } + for _, args := range s.Verify { + vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, args)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, args)...) + if vs.code != vl.code { + failf("verify %v: exit served %d, embedded %d: served %s", args, vs.code, vl.code, vs) + continue + } + if vl.code != 0 { + continue // both refused the read (e.g. a deleted id): equal is the check + } + if d := compareJSON(vs.stdout, vl.stdout, s.IgnoreFields, false); d != "" { + failf("verify %v diverged after the write: %s", args, d) + } + } + return fails +} + +func hasCorpusBug(failures []string) bool { + for _, f := range failures { + if strings.HasPrefix(f, "CORPUS BUG") { + return true + } + } + return false +} + +func modeOf(s shape) string { + if s.Compare == "" { + return "json" + } + return s.Compare +} + +func compareOutputs(s shape, served, local string) string { + switch modeOf(s) { + case "exit": + return "" + case "text": + a, b := scrubText(served), scrubText(local) + if a != b { + return fmt.Sprintf("text differs\nserved:\n%s\nembedded:\n%s", clip(a, 3000), clip(b, 3000)) + } + return "" + case "count": + a, errA := topLevelLen(served) + b, errB := topLevelLen(local) + if errA != nil || errB != nil { + return fmt.Sprintf("count: served err=%v embedded err=%v\nserved:\n%s", errA, errB, clip(served, 2000)) + } + if a != b { + return fmt.Sprintf("row count: served %d, embedded %d", a, b) + } + return "" + case "ids": + a, errA := idList(served, s.Ordered) + b, errB := idList(local, s.Ordered) + if errA != nil || errB != nil { + return fmt.Sprintf("ids: served err=%v embedded err=%v\nserved:\n%s", errA, errB, clip(served, 2000)) + } + if !reflect.DeepEqual(a, b) { + return fmt.Sprintf("ids differ\nserved (%d): %v\nembedded (%d): %v", len(a), a, len(b), b) + } + return "" + default: + return compareJSON(served, local, s.IgnoreFields, s.Ordered) + } +} + +// comparePrefix holds a limited listing to its unlimited form on each side: +// see shape.PrefixOf. +func comparePrefix(served, local, servedAll, localAll string) string { + lists := make([][]string, 4) + for i, out := range []string{served, local, servedAll, localAll} { + ids, err := idList(out, true) + if err != nil { + return fmt.Sprintf("listing %d: %v\n%s", i, err, clip(out, 2000)) + } + lists[i] = ids + } + s, l, sAll, lAll := lists[0], lists[1], lists[2], lists[3] + if len(s) != len(l) { + return fmt.Sprintf("row count: served %d, embedded %d", len(s), len(l)) + } + for _, c := range []struct { + name string + rows, all []string + }{{"served", s, sAll}, {"embedded", l, lAll}} { + if len(c.rows) > len(c.all) || !reflect.DeepEqual(c.rows, c.all[:len(c.rows)]) { + return fmt.Sprintf("%s limited rows are not a prefix of its own unlimited listing\nlimited: %v\nunlimited: %v", c.name, c.rows, c.all) + } + } + sSet := append([]string(nil), sAll...) + lSet := append([]string(nil), lAll...) + sort.Strings(sSet) + sort.Strings(lSet) + if !reflect.DeepEqual(sSet, lSet) { + return fmt.Sprintf("unlimited ids differ\nserved (%d): %v\nembedded (%d): %v", len(sSet), sSet, len(lSet), lSet) + } + return "" +} + +func scrubText(s string) string { + return strings.TrimSpace(timestampText.ReplaceAllString(s, "")) +} + +func decode(s string) (any, error) { + var v any + if err := json.Unmarshal([]byte(strings.TrimSpace(s)), &v); err != nil { + return nil, fmt.Errorf("not JSON: %w", err) + } + return v, nil +} + +// rows finds the row array in a bd --json payload: the payload itself, or the +// first array member of an envelope object. +func rows(v any) ([]any, bool) { + switch x := v.(type) { + case []any: + return x, true + case map[string]any: + for _, k := range []string{"issues", "data", "items", "results"} { + if arr, ok := x[k].([]any); ok { + return arr, true + } + } + } + return nil, false +} + +func topLevelLen(s string) (int, error) { + v, err := decode(s) + if err != nil { + return 0, err + } + arr, ok := rows(v) + if !ok { + return 0, fmt.Errorf("no row array in output") + } + return len(arr), nil +} + +func idList(s string, ordered bool) ([]string, error) { + v, err := decode(s) + if err != nil { + return nil, err + } + arr, ok := rows(v) + if !ok { + return nil, fmt.Errorf("no row array in output") + } + out := []string{} + for _, r := range arr { + if m, ok := r.(map[string]any); ok { + if id, ok := m["id"].(string); ok { + out = append(out, id) + } + } + } + if !ordered { + sort.Strings(out) + } + return out, nil +} + +func compareJSON(served, local string, ignore []string, ordered bool) string { + a, errA := decode(served) + b, errB := decode(local) + if errA != nil || errB != nil { + return fmt.Sprintf("served err=%v embedded err=%v\nserved:\n%s\nembedded:\n%s", errA, errB, clip(served, 2000), clip(local, 2000)) + } + drop := map[string]bool{} + for _, k := range ignore { + drop[k] = true + } + a, b = normalize(a, drop, ordered), normalize(b, drop, ordered) + if path, x, y, same := firstDiff("$", a, b); !same { + return fmt.Sprintf("first difference at %s: served=%s embedded=%s", path, brief(x), brief(y)) + } + return "" +} + +func normalize(v any, drop map[string]bool, ordered bool) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + if drop[k] { + continue + } + if callerTimeKey[k] { + out[k] = val + continue + } + if volatileKey.MatchString(k) { + continue + } + out[k] = normalize(val, drop, ordered) + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = normalize(e, drop, ordered) + } + if !ordered { + sort.SliceStable(out, func(i, j int) bool { return sortKey(out[i]) < sortKey(out[j]) }) + } + return out + case string: + return timestampText.ReplaceAllString(x, "") + default: + return v + } +} + +func sortKey(v any) string { + if m, ok := v.(map[string]any); ok { + for _, k := range []string{"id", "issue_id", "key"} { + if s, ok := m[k].(string); ok { + return k + "=" + s + } + } + if inner, ok := m["issue"].(map[string]any); ok { + if s, ok := inner["id"].(string); ok { + return "issue.id=" + s + } + } + } + raw, _ := json.Marshal(v) + return string(raw) +} + +func firstDiff(path string, a, b any) (string, any, any, bool) { + switch x := a.(type) { + case map[string]any: + y, ok := b.(map[string]any) + if !ok { + return path, a, b, false + } + keys := map[string]bool{} + for k := range x { + keys[k] = true + } + for k := range y { + keys[k] = true + } + sorted := make([]string, 0, len(keys)) + for k := range keys { + sorted = append(sorted, k) + } + sort.Strings(sorted) + for _, k := range sorted { + xv, xok := x[k] + yv, yok := y[k] + if !xok || !yok { + return path + "." + k, xv, yv, false + } + if p, l, r, same := firstDiff(path+"."+k, xv, yv); !same { + return p, l, r, false + } + } + return "", nil, nil, true + case []any: + y, ok := b.([]any) + if !ok { + return path, a, b, false + } + if len(x) != len(y) { + return path + ".length", len(x), len(y), false + } + for i := range x { + if p, l, r, same := firstDiff(path+"["+strconv.Itoa(i)+"]", x[i], y[i]); !same { + return p, l, r, false + } + } + return "", nil, nil, true + default: + if reflect.DeepEqual(a, b) { + return "", nil, nil, true + } + return path, a, b, false + } +} + +func brief(v any) string { + if v == nil { + return "" + } + raw, _ := json.Marshal(v) + return clip(string(raw), 400) +} + +var placeholder = regexp.MustCompile(`\{(file|rev):([^}]+)\}`) + +// expand substitutes {file:NAME} and {rev:ID} in args for side sd. +func expand(t *testing.T, bin string, sd *side, args []string) []string { + t.Helper() + out := make([]string, len(args)) + for i, a := range args { + out[i] = placeholder.ReplaceAllStringFunc(a, func(m string) string { + parts := placeholder.FindStringSubmatch(m) + switch parts[1] { + case "file": + return filepath.Join(sd.files, parts[2]) + default: + r := runBD(t, bin, sd.dir, sd.env, "", "show", parts[2], "--json") + if r.code != 0 { + t.Logf("{rev:%s} on %s: show failed: %s", parts[2], sd.name, r) + return "0" + } + v, err := decode(r.stdout) + if err != nil { + return "0" + } + if arr, ok := rows(v); ok && len(arr) > 0 { + v = arr[0] + } + if m, ok := v.(map[string]any); ok { + if rev, ok := m["revision"].(string); ok { + return rev + } + } + return "0" + } + }) + } + return out +} + +func writeAssets(t *testing.T, assets []corpusAsset, dir string) { + t.Helper() + for _, a := range assets { + content := a.Content + if a.Bulk != nil { + content = bulkPlan(*a.Bulk) + } + if err := os.WriteFile(filepath.Join(dir, a.Name), []byte(content), 0o600); err != nil { + t.Fatalf("write asset %s: %v", a.Name, err) + } + } +} + +func bulkPlan(b bulkGraph) string { + type node struct { + Key string `json:"key"` + ID string `json:"id"` + Title string `json:"title"` + Type string `json:"type"` + Priority int `json:"priority"` + Ephemeral bool `json:"ephemeral,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + } + plan := struct { + CommitMessage string `json:"commit_message"` + Nodes []node `json:"nodes"` + }{CommitMessage: "corpus bulk " + b.IDPrefix} + for i := 1; i <= b.Count; i++ { + id := fmt.Sprintf("%s-%03d", b.IDPrefix, i) + plan.Nodes = append(plan.Nodes, node{Key: id, ID: id, Title: "bulk " + id, Type: "task", + Priority: b.Priority, Ephemeral: b.Ephemeral, Metadata: b.Metadata}) + } + raw, _ := json.MarshalIndent(plan, "", " ") + return string(raw) +} + +func summarize(t *testing.T, outcomes []outcome) { + t.Helper() + counts := map[string]int{} + var lines []string + for _, o := range outcomes { + key := o.status + if strings.HasPrefix(key, "pending:") { + key = "pending" + } + counts[key]++ + lines = append(lines, fmt.Sprintf("%-26s %-48s req=%-3d %s", o.status, o.id, o.requests, firstLine(o.detail))) + } + t.Logf("corpus summary: %v\n%s", counts, strings.Join(lines, "\n")) + if path := os.Getenv("BEADS_CORPUS_REPORT"); path != "" { + type row struct { + ID string `json:"id"` + Status string `json:"status"` + Requests int64 `json:"requests"` + Detail string `json:"detail,omitempty"` + } + var out []row + for _, o := range outcomes { + out = append(out, row{o.id, o.status, o.requests, o.detail}) + } + raw, _ := json.MarshalIndent(out, "", " ") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Errorf("write BEADS_CORPUS_REPORT: %v", err) + } + } +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return clip(s, 160) +} diff --git a/backend/http/corpus/harness_test.go b/backend/http/corpus/harness_test.go new file mode 100644 index 000000000..3b93d0473 --- /dev/null +++ b/backend/http/corpus/harness_test.go @@ -0,0 +1,312 @@ +//go:build cgo + +package corpus_test + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +const ( + corpusProjectID = "proj-corpus-s23" + corpusDatabase = "corpus" + corpusPrefix = "e2e" + // corpusToken is the bearer the in-process server accepts. The reference + // deployment authenticates every request, so the suite does too. + // #nosec G101 -- a throwaway test credential for an httptest-scoped server. + corpusToken = "corpus-s23-test-token" + // bdTimeout bounds one bd subprocess so a hung shape fails rather than + // stalling the whole suite. + bdTimeout = 90 * time.Second +) + +// startCorpusServer binds the in-process server the way the reference +// deployment is bound: AllowNonLoopback with a bearer token file. The socket is +// still 127.0.0.1 (this suite never leaves the host), but every behavior the +// server keys off its bind mode — the loopback-only refusal of an unlimited +// read above all — is the non-loopback one, because that is decided by +// Config.AllowNonLoopback and not by the address. +func startCorpusServer(t *testing.T) string { + t.Helper() + beadsDir := t.TempDir() + ctx := context.Background() + store, err := embeddeddolt.Open(ctx, beadsDir, corpusDatabase, "main") + if err != nil { + t.Fatalf("open the served store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", corpusPrefix); err != nil { + t.Fatalf("set the issue prefix: %v", err) + } + + tokenFile := filepath.Join(t.TempDir(), "tokens") + if err := os.WriteFile(tokenFile, []byte(corpusToken+"\n"), 0o600); err != nil { + t.Fatalf("write the token file: %v", err) + } + auth, err := httpapi.NewTokenFileAuth(tokenFile) + if err != nil { + t.Fatalf("load the token file: %v", err) + } + + cfg := corpusServeConfig(t, store) + cfg.AllowNonLoopback = true + cfg.Auth = auth + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the in-process server: %v", err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("the in-process server did not shut down") + } + }) + return srv.Addr() +} + +// startProxy serves h on an ephemeral loopback port. +func startProxy(t *testing.T, h http.Handler) string { + t.Helper() + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen for the reference proxy: %v", err) + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return ln.Addr().String() +} + +// corpusServeConfig takes every role off the store, as bd serve does. It +// mirrors backend/http's e2eServeConfig (a sibling test package, so it cannot +// be imported). +func corpusServeConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { + t.Helper() + cfg := httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + Workspace: domain.ContextInfo{ProjectID: corpusProjectID, Database: corpusDatabase}, + } + var err error + must := func(name string) { + t.Helper() + if err != nil { + t.Fatalf("%s(): %v", name, err) + } + } + cfg.Reader, err = s.IssueReader() + must("IssueReader") + cfg.Claimer, err = s.IssueClaimer() + must("IssueClaimer") + cfg.ReadyClaimer, err = s.ReadyClaimer() + must("ReadyClaimer") + cfg.Releaser, err = s.Releaser() + must("Releaser") + cfg.Lifecycle, err = s.IssueLifecycle() + must("IssueLifecycle") + cfg.Settings, err = s.WorkspaceConfig() + must("WorkspaceConfig") + cfg.Stats, err = s.StatsReporter() + must("StatsReporter") + cfg.CycleDetector, err = s.CycleDetector() + must("CycleDetector") + cfg.EdgeReader, err = s.EdgeReader() + must("EdgeReader") + cfg.GraphCounter, err = s.GraphCounter() + must("GraphCounter") + cfg.BatchGetter, err = s.BatchGetter() + must("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + must("LeaseReclaimer") + cfg.Relations, err = s.IssueRelations() + must("IssueRelations") + cfg.Commenter, err = s.Commenter() + must("Commenter") + cfg.BlockingAnnotator, err = s.BlockingAnnotator() + must("BlockingAnnotator") + cfg.TreeWalker, err = s.TreeWalker() + must("TreeWalker") + cfg.ReadyCounter, err = s.ReadyCounter() + must("ReadyCounter") + cfg.Counter, err = s.Counter() + must("Counter") + cfg.Querier, err = s.Querier() + must("Querier") + cfg.Sweeper, err = s.Sweeper() + must("Sweeper") + cfg.Deleter, err = s.Deleter() + must("Deleter") + cfg.BatchCreator, err = s.BatchCreator() + must("BatchCreator") + cfg.BatchCloser, err = s.BatchCloser() + must("BatchCloser") + cfg.DependencyEditor, err = s.DependencyEditor() + must("DependencyEditor") + cfg.MetadataCAS, err = s.MetadataCAS() + must("MetadataCAS") + cfg.BatchApplier, err = s.BatchApplier() + must("BatchApplier") + cfg.Memories, err = s.Memories() + must("Memories") + return cfg +} + +// This file is the corpus tier's cgo half; see TestCorpusTierIsLinkedWhenRequired. +func init() { corpusTierLinked = true } + +// skipUnlessEmbeddedDolt is the served tier's gate, as in backend/http. +func skipUnlessEmbeddedDolt(t *testing.T) { + t.Helper() + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") == "1" { + return + } + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but BEADS_TEST_EMBEDDED_DOLT is not 1; the http corpus tier is not enforced") + } + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run the http command-corpus tier") +} + +type bdResult struct { + stdout, stderr string + code int +} + +func (r bdResult) String() string { + return fmt.Sprintf("exit=%d\nstdout:\n%s\nstderr:\n%s", r.code, clip(r.stdout, 4000), clip(r.stderr, 4000)) +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + fmt.Sprintf("\n... (%d more bytes)", len(s)-n) +} + +// runBD runs the built bd binary in dir with a hermetic environment. +func runBD(t *testing.T, bin, dir string, env []string, stdin string, args ...string) bdResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), bdTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Dir = dir + cmd.Env = minimalBDEnv(dir, env...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var o, e strings.Builder + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + code := 0 + if err != nil { + var ee *exec.ExitError + switch { + case ctx.Err() != nil: + code = -1 + e.WriteString(fmt.Sprintf("\n[corpus harness] bd timed out after %s", bdTimeout)) + case errors.As(err, &ee): + code = ee.ExitCode() + default: + t.Fatalf("running bd %s: %v", strings.Join(args, " "), err) + } + } + return bdResult{stdout: o.String(), stderr: e.String(), code: code} +} + +// minimalBDEnv is backend/http's hermetic bd environment: PATH and the few +// variables a Go binary needs, HOME repointed at dir, nothing BEADS_* carried +// over from the invoking shell. extra is applied last. +func minimalBDEnv(dir string, extra ...string) []string { + env := []string{ + "HOME=" + dir, + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + } + for _, key := range []string{"PATH", "TMPDIR", "SYSTEMROOT", "WINDIR"} { + if v, ok := os.LookupEnv(key); ok { + env = append(env, key+"="+v) + } + } + return append(env, extra...) +} + +var ( + buildBDOnce sync.Once + bdBinPath string + bdBuildErr error + bdBinOwnBuild bool +) + +// buildBD builds bd from this tree once per test process (or takes Bazel's +// prebuilt binary). TestMain deletes the build. +func buildBD(t *testing.T) string { + t.Helper() + buildBDOnce.Do(func() { + if bazeltest.IsBazel() { + bdBinPath, bdBuildErr = bazeltest.PrebuiltBD() + return + } + if prebuilt, err := bazeltest.PrebuiltBD(); err != nil || prebuilt != "" { + bdBinPath, bdBuildErr = prebuilt, err + return + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "bd-corpus") + if err != nil { + bdBuildErr = fmt.Errorf("mkdir temp for bd binary: %w", err) + return + } + bin := filepath.Join(dir, "bd-corpus") + cmd := exec.Command("go", "build", "-tags", "gms_pure_go", "-o", bin, "./cmd/bd") + cmd.Dir = repoRoot() + if out, err := cmd.CombinedOutput(); err != nil { + bdBuildErr = fmt.Errorf("build bd: %w\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + bdBinPath = bin + bdBinOwnBuild = true + }) + if bdBuildErr != nil { + t.Fatal(bdBuildErr) + } + if bdBinPath == "" { + t.Fatal("no bd binary: Bazel did not provide one") + } + return bdBinPath +} + +func TestMain(m *testing.M) { + code := m.Run() + if bdBinOwnBuild && bdBinPath != "" { + _ = os.RemoveAll(filepath.Dir(bdBinPath)) + } + os.Exit(code) +} + +func repoRoot() string { + _, file, _, _ := runtime.Caller(0) // /backend/http/corpus/harness_test.go + return filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(file)))) +} diff --git a/backend/http/corpus/reference_proxy_test.go b/backend/http/corpus/reference_proxy_test.go new file mode 100644 index 000000000..0142b1ad4 --- /dev/null +++ b/backend/http/corpus/reference_proxy_test.go @@ -0,0 +1,279 @@ +//go:build cgo + +package corpus_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + "net/http/httputil" + "net/url" + "os" + "regexp" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// referenceProfile is testdata/reference_server.json: the wire surface of the +// deployed server the corpus must work against. See README.md. +type referenceProfile struct { + Capabilities []string `json:"capabilities"` + ContextMembers []string `json:"context_members"` + Operations []referenceOperation `json:"operations"` +} + +type referenceOperation struct { + Method string `json:"method"` + Path string `json:"path"` + OperationID string `json:"operation_id"` + Query []string `json:"query"` + Body []string `json:"body"` + + pattern *regexp.Regexp + query map[string]bool + body map[string]bool +} + +func loadReferenceProfile(t *testing.T) *referenceProfile { + t.Helper() + raw, err := os.ReadFile("testdata/reference_server.json") + if err != nil { + t.Fatalf("read the reference server profile: %v", err) + } + var p referenceProfile + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatalf("parse testdata/reference_server.json: %v", err) + } + if len(p.Capabilities) == 0 || len(p.Operations) == 0 || len(p.ContextMembers) == 0 { + t.Fatal("testdata/reference_server.json is missing capabilities, operations or context_members") + } + for i := range p.Operations { + op := &p.Operations[i] + pattern, err := templatePattern(op.Path) + if err != nil { + t.Fatalf("testdata/reference_server.json: operation %s path %q: %v", op.OperationID, op.Path, err) + } + op.pattern = pattern + op.query = setOf(op.Query) + op.body = setOf(op.Body) + } + return &p +} + +// templatePattern turns an OpenAPI path template into an anchored regexp. A +// path parameter never spans a "/" or a ":" (AIP-136 custom methods hang off +// the last segment as {id}:verb, and no issue id, config key or memory key +// carries a raw colon), so /issues/{id} does not swallow /issues/{id}:claim. +// A "{" with no "}" is an error: the profile is edited by hand, and without the +// check the loop would never advance past it. +func templatePattern(tmpl string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for { + open := strings.IndexByte(tmpl, '{') + if open < 0 { + b.WriteString(regexp.QuoteMeta(tmpl)) + break + } + closeAt := strings.IndexByte(tmpl[open:], '}') + if closeAt < 0 { + return nil, fmt.Errorf("unclosed { in %q", tmpl[open:]) + } + b.WriteString(regexp.QuoteMeta(tmpl[:open])) + b.WriteString(`[^/:]+`) + tmpl = tmpl[open+closeAt+1:] + } + b.WriteString("$") + return regexp.Compile(b.String()) +} + +func setOf(xs []string) map[string]bool { + m := make(map[string]bool, len(xs)) + for _, x := range xs { + m[x] = true + } + return m +} + +// referenceProxy fronts the in-process server and makes it answer the way the +// reference server answers, at the wire level, for everything this suite can +// see from a client: +// +// - GET /v0/beads/context publishes only the reference ContextResponse +// members (so no wire_revision: the reference server predates it) and +// exactly the reference capability set, whatever this build advertises; +// - a request for an operation the reference document does not publish gets +// the server's own unrouted 404 problem; +// - a query parameter or top-level JSON request member the reference +// operation does not declare gets the server's own 400 unknown_parameter +// problem naming it — which is how the real reference server answers, +// since its decoder is strict. +// +// Responses are otherwise passed through untouched. That is deliberately +// generous in one direction (a response member the reference server would not +// send still reaches the client) and is the residual fidelity gap README.md +// records. +type referenceProxy struct { + profile *referenceProfile + upstream *url.URL + rp *httputil.ReverseProxy + + requests atomic.Int64 + + mu sync.Mutex + refused []string // "METHOD path: why", for the per-shape log +} + +func newReferenceProxy(t *testing.T, profile *referenceProfile, upstreamAddr string) *referenceProxy { + t.Helper() + u, err := url.Parse("http://" + upstreamAddr) + if err != nil { + t.Fatalf("parse upstream address: %v", err) + } + p := &referenceProxy{profile: profile, upstream: u} + p.rp = &httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + r.SetURL(u) // also rewrites Host, so the server's Host gate sees its own bind + }, + FlushInterval: -1, // events:watch streams + } + return p +} + +func (p *referenceProxy) takeRefusals() []string { + p.mu.Lock() + defer p.mu.Unlock() + out := p.refused + p.refused = nil + return out +} + +func (p *referenceProxy) noteRefusal(r *http.Request, why string) { + p.mu.Lock() + defer p.mu.Unlock() + p.refused = append(p.refused, fmt.Sprintf("%s %s: %s", r.Method, r.URL.Path, why)) +} + +func (p *referenceProxy) match(r *http.Request) *referenceOperation { + path := r.URL.EscapedPath() + var best *referenceOperation + for i := range p.profile.Operations { + op := &p.profile.Operations[i] + if op.Method != r.Method || !op.pattern.MatchString(path) { + continue + } + // Prefer the most literal template when two match (none do today). + if best == nil || len(op.Path) > len(best.Path) { + best = op + } + } + return best +} + +func (p *referenceProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p.requests.Add(1) + if r.Method == http.MethodGet && r.URL.Path == "/v0/beads/context" { + p.serveContext(w, r) + return + } + op := p.match(r) + if op == nil { + p.noteRefusal(r, "operation not published by the reference server") + detail := "no such route on this server" + httpapi.Write(w, httpapi.Result{Problem: apigen.Problem{ + Status: http.StatusNotFound, + Title: http.StatusText(http.StatusNotFound), + Code: string(httpapi.CodeNotFound), + Detail: &detail, + }}) + return + } + if offender := firstUnknown(slices.Collect(maps.Keys(r.URL.Query())), op.query); offender != "" { + p.noteRefusal(r, "query parameter "+offender+" unknown to "+op.OperationID) + httpapi.Write(w, httpapi.InvalidArgument(offender, httpapi.ReasonUnknownParameter, + "unknown query parameter "+offender)) + return + } + if r.Body != nil && r.Body != http.NoBody { + raw, err := io.ReadAll(r.Body) + _ = r.Body.Close() + if err != nil { + http.Error(w, "reference proxy: read body: "+err.Error(), http.StatusBadGateway) + return + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + r.ContentLength = int64(len(raw)) + var members map[string]json.RawMessage + if len(bytes.TrimSpace(raw)) > 0 && json.Unmarshal(raw, &members) == nil { + if offender := firstUnknown(slices.Collect(maps.Keys(members)), op.body); offender != "" { + p.noteRefusal(r, "request member "+offender+" unknown to "+op.OperationID) + httpapi.Write(w, httpapi.InvalidArgument(offender, httpapi.ReasonUnknownParameter, + "json: unknown field \""+offender+"\"")) + return + } + } + } + p.rp.ServeHTTP(w, r) +} + +// firstUnknown names the smallest key not in known, the same deterministic +// choice the server makes, or "" when every key is known. +func firstUnknown(keys []string, known map[string]bool) string { + sort.Strings(keys) + for _, k := range keys { + if !known[k] { + return k + } + } + return "" +} + +func (p *referenceProxy) serveContext(w http.ResponseWriter, r *http.Request) { + out := r.Clone(r.Context()) + out.URL.Scheme, out.URL.Host, out.Host = p.upstream.Scheme, p.upstream.Host, p.upstream.Host + out.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(out) + if err != nil { + http.Error(w, "reference proxy: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, "reference proxy: "+err.Error(), http.StatusBadGateway) + return + } + if resp.StatusCode != http.StatusOK { + maps.Copy(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(raw) + return + } + var body map[string]json.RawMessage + if err := json.Unmarshal(raw, &body); err != nil { + http.Error(w, "reference proxy: decode context: "+err.Error(), http.StatusBadGateway) + return + } + keep := setOf(p.profile.ContextMembers) + for k := range body { + if !keep[k] { + delete(body, k) + } + } + caps, _ := json.Marshal(p.profile.Capabilities) + body["capabilities"] = caps + masked, _ := json.Marshal(body) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(masked) +} diff --git a/backend/http/corpus/required_corpus_tier_test.go b/backend/http/corpus/required_corpus_tier_test.go new file mode 100644 index 000000000..055afeb79 --- /dev/null +++ b/backend/http/corpus/required_corpus_tier_test.go @@ -0,0 +1,23 @@ +package corpus_test + +import ( + "os" + "testing" +) + +// corpusTierLinked reports whether this test binary links the corpus tier. +// harness_test.go, which builds only under cgo like every other file in this +// package, sets it from its init. +var corpusTierLinked bool + +// TestCorpusTierIsLinkedWhenRequired is the one corpus check that builds +// without cgo. skipUnlessEmbeddedDolt turns a missing BEADS_TEST_EMBEDDED_DOLT +// into a failure under BEADS_HTTP_TEST_REQUIRED=1, but it lives in a cgo file: +// with cgo off the go tool drops every other file in this package, so no +// corpus test exists to fail, and a required run would pass having run nothing. +func TestCorpusTierIsLinkedWhenRequired(t *testing.T) { + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" && !corpusTierLinked { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but this test binary was built without cgo, which drops " + + "the http command corpus (every other file in this package); build it with CGO_ENABLED=1") + } +} diff --git a/backend/http/corpus/testdata/allowlist.yaml b/backend/http/corpus/testdata/allowlist.yaml new file mode 100644 index 000000000..090577cc9 --- /dev/null +++ b/backend/http/corpus/testdata/allowlist.yaml @@ -0,0 +1,29 @@ +# Shapes that are EXPECTED to be refused on a served workspace, permanently. +# The served run must fail and its output must match `refusal`; if it ever +# succeeds, the entry is stale and the suite fails until it is removed. The +# embedded run is still held to the shape's expect_exit, which an allowlisted +# shape must state, and to its expect_stderr: a shape that stops doing what it +# says on the embedded backend fails as a corpus bug, not as allowed. +# A gap some slice will close is NOT listed here: it is a `pending` mark on the +# shape in corpus.yaml instead. + +- id: gc-sql-ready-projection + reason: >- + bd sql needs a Dolt sql-server workspace: the embedded backend refuses it + too (the shape's expect_exit and expect_stderr pin that), and an http + workspace refuses it with the same "embedded mode" text. gc must not issue + it against a remote workspace (DESIGN B3 #10 and B6: gc reads the ready + projection through roles there). + refusal: "'bd sql' is not yet supported" +- id: gc-sql-release-fallback + reason: >- + bd sql needs a Dolt sql-server workspace and the embedded backend refuses + it too; gc's ReleaseIfCurrent fallback must take the conditional update + path on a remote workspace (DESIGN B3 #10). + refusal: "'bd sql' is not yet supported" +- id: mc-meta-set + reason: >- + `bd meta set` is not an OSS bd command (pack scripts written against an + older pinned CLI); it fails on the embedded backend too. The packs should + use `bd update --set-metadata`, which the corpus covers. + refusal: 'unknown command "meta"' diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml new file mode 100644 index 000000000..ded7e9e30 --- /dev/null +++ b/backend/http/corpus/testdata/corpus.yaml @@ -0,0 +1,735 @@ +# Served command corpus (slice S23): every distinct bd command shape the +# orchestrator (gc) and its packs/formulas issue against a work store. +# +# Sources: +# gc: gascity origin/main internal/beads/bdstore*.go, internal/config/workquery.go, +# cmd/gc (BdStore subprocess shapes, work_query/scale_check/on_boot/on_death +# shell, hook claim, agent-script actions, doctor/preflight probes). +# mc: pack, formula, prompt and script invocations counted across the +# maintainer packs (flag sets, not values; values here are representative). +# +# Each shape runs on a served workspace (bd connect to an in-process bd serve, +# non-loopback bind mode, bearer auth, masked to testdata/reference_server.json) +# and on an embedded workspace, with BEADS_ACTOR=worker and CLAUDE_SESSION_ID set +# on both. See README.md for the fields. Reads come first; writes after them +# each own their target ids (e2e-t-*), so write order never changes a read. +# +# pending: {slice, reason} marks a known failure. The shape still runs; when it +# passes the subtest FAILS until the mark is removed. That is how a slice proves +# itself. Permanent refusals live in allowlist.yaml instead. + +files: + - name: seed-core.json + content: | + {"commit_message": "corpus seed", "nodes": [ + {"key": "r1", "id": "e2e-r1", "title": "Routed open task", "type": "task", "priority": 1, "labels": ["lane:a"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "r2", "id": "e2e-r2", "title": "Held routed task", "type": "task", "labels": ["hold:mayor"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "r3", "id": "e2e-r3", "title": "Routed epic", "type": "epic", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "wf1", "id": "e2e-wf1", "title": "Workflow root", "type": "task", "metadata": {"gc.run_target": "pool-a", "gc.kind": "workflow"}}, + {"key": "a1", "id": "e2e-a1", "title": "Alice bug", "type": "bug", "assignee": "alice"}, + {"key": "ip1", "id": "e2e-ip1", "title": "Worker in progress", "type": "task", "priority": 0, "assignee": "worker", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "wr", "id": "e2e-wr", "title": "Worker ready task", "type": "task", "assignee": "worker"}, + {"key": "ipna", "id": "e2e-ipna", "title": "Orphaned in progress", "type": "task", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "ipnb", "id": "e2e-ipnb", "title": "Orphaned workflow", "type": "task", "metadata": {"gc.run_target": "pool-a", "gc.kind": "workflow"}}, + {"key": "blk", "id": "e2e-blk", "title": "Blocked by r1", "type": "task", "deps": [{"target": "r1"}]}, + {"key": "done", "id": "e2e-done", "title": "Closed task", "type": "task"}, + {"key": "ext", "id": "e2e-ext", "title": "GitHub source", "type": "task", "external_ref": "gh-123", "metadata": {"gc.kind": "github_source", "gc.github.kind": "issue", "gc.github.repo": "o/r"}}, + {"key": "ep", "id": "e2e-ep", "title": "Epic", "type": "epic"}, + {"key": "ep.1", "id": "e2e-ep.1", "title": "Epic child one", "type": "task", "parent_key": "ep"}, + {"key": "ep.2", "id": "e2e-ep.2", "title": "Epic child two", "type": "task", "parent_key": "ep"}, + {"key": "mol", "id": "e2e-mol", "title": "Molecule root", "type": "epic"}, + {"key": "mol.1", "id": "e2e-mol.1", "title": "step one", "type": "task", "assignee": "worker", "parent_key": "mol"}, + {"key": "mol.2", "id": "e2e-mol.2", "title": "step two", "type": "task", "parent_key": "mol", "deps": [{"target": "mol.1"}], "metadata": {"gc.root_bead_id": "e2e-mol", "gc.routed_to": "pool-a"}}, + {"key": "mol.3", "id": "e2e-mol.3", "title": "step three", "type": "task", "parent_key": "mol"}, + {"key": "mac", "id": "e2e-mac", "title": "Auto-close molecule", "type": "epic"}, + {"key": "mac.1", "id": "e2e-mac.1", "title": "done step", "type": "task", "parent_key": "mac"}, + {"key": "mac.2", "id": "e2e-mac.2", "title": "last step", "type": "task", "assignee": "worker", "parent_key": "mac"}, + {"key": "wisp-1", "id": "e2e-wisp-1", "title": "Wisp wisp-1", "type": "task", "ephemeral": true, "metadata": {"gc.routed_to": "pool-a"}, "labels": ["lane:w"]}, + {"key": "wisp-2", "id": "e2e-wisp-2", "title": "Wisp wisp-2", "type": "task", "ephemeral": true, "assignee": "worker", "labels": ["lane:w"]}, + {"key": "wisp-wr", "id": "e2e-wisp-wr", "title": "Wisp worker ready", "type": "task", "ephemeral": true, "priority": 1, "assignee": "worker"}, + {"key": "wisp-c", "id": "e2e-wisp-c", "title": "Wisp wisp-c", "type": "task", "ephemeral": true}, + {"key": "wisp-p", "id": "e2e-wisp-p", "title": "Wisp wisp-p", "type": "task", "ephemeral": true}, + {"key": "wisp-m", "id": "e2e-wisp-m", "title": "Wisp molecule", "type": "epic", "ephemeral": true}, + {"key": "wisp-m.1", "id": "e2e-wisp-m.1", "title": "wisp step", "type": "task", "ephemeral": true, "parent_key": "wisp-m"}, + {"key": "t-upd", "id": "e2e-t-upd", "title": "target upd", "type": "task", "labels": ["lane:z"]}, + {"key": "t-upd2", "id": "e2e-t-upd2", "title": "target upd2", "type": "task"}, + {"key": "t-m1", "id": "e2e-t-m1", "title": "target m1", "type": "task"}, + {"key": "t-m2", "id": "e2e-t-m2", "title": "target m2", "type": "task"}, + {"key": "t-claim", "id": "e2e-t-claim", "title": "target claim", "type": "task"}, + {"key": "t-claim2", "id": "e2e-t-claim2", "title": "target claim2", "type": "task"}, + {"key": "t-rev", "id": "e2e-t-rev", "title": "target rev", "type": "task"}, + {"key": "t-rev2", "id": "e2e-t-rev2", "title": "target rev2", "type": "task"}, + {"key": "t-sling", "id": "e2e-t-sling", "title": "target sling", "type": "task"}, + {"key": "t-as", "id": "e2e-t-as", "title": "target as", "type": "task"}, + {"key": "t-ms", "id": "e2e-t-ms", "title": "target ms", "type": "task"}, + {"key": "t-msc", "id": "e2e-t-msc", "title": "target msc", "type": "task"}, + {"key": "t-um", "id": "e2e-t-um", "title": "target um", "type": "task", "metadata": {"k": "v"}}, + {"key": "t-notes", "id": "e2e-t-notes", "title": "target notes", "type": "task"}, + {"key": "t-er", "id": "e2e-t-er", "title": "target er", "type": "task"}, + {"key": "t-md", "id": "e2e-t-md", "title": "target md", "type": "task"}, + {"key": "t-asg", "id": "e2e-t-asg", "title": "target asg", "type": "task"}, + {"key": "t-dac", "id": "e2e-t-dac", "title": "target dac", "type": "task"}, + {"key": "t-c1", "id": "e2e-t-c1", "title": "target c1", "type": "task"}, + {"key": "t-c2", "id": "e2e-t-c2", "title": "target c2", "type": "task"}, + {"key": "t-c3", "id": "e2e-t-c3", "title": "target c3", "type": "task"}, + {"key": "t-c4", "id": "e2e-t-c4", "title": "target c4", "type": "task"}, + {"key": "t-c5", "id": "e2e-t-c5", "title": "target c5", "type": "task"}, + {"key": "t-d1", "id": "e2e-t-d1", "title": "target d1", "type": "task"}, + {"key": "t-d2", "id": "e2e-t-d2", "title": "target d2", "type": "task"}, + {"key": "t-d3", "id": "e2e-t-d3", "title": "target d3", "type": "task"}, + {"key": "t-d4", "id": "e2e-t-d4", "title": "target d4", "type": "task"}, + {"key": "t-df1", "id": "e2e-t-df1", "title": "target df1", "type": "task"}, + {"key": "t-df2", "id": "e2e-t-df2", "title": "target df2", "type": "task"}, + {"key": "t-da", "id": "e2e-t-da", "title": "target da", "type": "task"}, + {"key": "t-db", "id": "e2e-t-db", "title": "target db", "type": "task"}, + {"key": "t-l", "id": "e2e-t-l", "title": "target l", "type": "task"}, + {"key": "t-as2", "id": "e2e-t-as2", "title": "target as2", "type": "task"}, + {"key": "t-cm", "id": "e2e-t-cm", "title": "target cm", "type": "task"}, + {"key": "t-rel", "id": "e2e-t-rel", "title": "target rel", "type": "task", "assignee": "worker"}, + {"key": "t-xfer", "id": "e2e-t-xfer", "title": "target xfer", "type": "task", "assignee": "worker"}, + {"key": "t-ifa", "id": "e2e-t-ifa", "title": "target ifa", "type": "task", "assignee": "worker"}, + {"key": "t-death", "id": "e2e-t-death", "title": "target death", "type": "task", "assignee": "worker"}, + {"key": "t-rc", "id": "e2e-t-rc", "title": "target rc", "type": "task", "assignee": "worker"}, + {"key": "t-boot", "id": "e2e-t-boot", "title": "target boot", "type": "task"}, + {"key": "t-ro", "id": "e2e-t-ro", "title": "target reopen", "type": "task"}, + {"key": "t-dr", "id": "e2e-t-dr", "title": "target dep remove", "type": "task", "deps": [{"target": "t-db"}]}, + {"key": "t-dc", "id": "e2e-t-dc", "title": "cascade root", "type": "epic"}, + {"key": "t-dc.1", "id": "e2e-t-dc.1", "title": "cascade child", "type": "task", "parent_key": "t-dc"} + ]} + - name: bulk-ready-a.json + bulk: {id_prefix: e2e-bra, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} + - name: bulk-ready-b.json + bulk: {id_prefix: e2e-brb, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} + - name: bulk-wisp-a.json + bulk: {id_prefix: e2e-wisp-bwa, count: 55, priority: 3, ephemeral: true, metadata: {gc.routed_to: pool-a}} + - name: bulk-wisp-b.json + bulk: {id_prefix: e2e-wisp-bwb, count: 55, priority: 3, ephemeral: true, metadata: {gc.routed_to: pool-a}} + - name: graph-small.json + content: | + {"commit_message": "gc graph apply", "nodes": [ + {"key": "root", "id": "e2e-g-root", "title": "graph root", "type": "epic", "metadata": {"gc.kind": "workflow"}}, + {"key": "s1", "id": "e2e-g-s1", "title": "graph step 1", "type": "task", "parent_key": "root", "assignee": "worker", "labels": ["lane:g"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "s2", "id": "e2e-g-s2", "title": "graph step 2", "type": "task", "parent_key": "root"} + ], "edges": [{"from_key": "s2", "to_key": "s1", "type": "blocks"}]} + - name: graph-eph.json + content: | + {"nodes": [ + {"key": "root", "id": "e2e-wisp-g-root", "title": "wisp graph root", "type": "epic"}, + {"key": "s1", "id": "e2e-wisp-g-s1", "title": "wisp graph step", "type": "task", "parent_key": "root"} + ]} + - name: delete-list.txt + content: | + e2e-t-df1 + e2e-t-df2 + +seed: + - [create, --graph, "{file:seed-core.json}", --json] + - [create, --graph, "{file:bulk-ready-a.json}", --json] + - [create, --graph, "{file:bulk-ready-b.json}", --json] + - [create, --graph, "{file:bulk-wisp-a.json}", --json] + - [create, --graph, "{file:bulk-wisp-b.json}", --json] + # Graph plans carry no status: a node created in_progress or closed stamps + # started_at/closed_at, which batchApply's create vocabulary cannot carry + # (gc's own plans never set a status). Lifecycle goes through update/close. + - [update, e2e-ip1, e2e-ipna, e2e-ipnb, e2e-mol.1, e2e-mac.2, e2e-t-rel, e2e-t-xfer, e2e-t-ifa, e2e-t-death, e2e-t-rc, e2e-t-boot, --status, in_progress] + - [update, e2e-wisp-2, --status, in_progress] + - [close, --force, e2e-done, e2e-ep.2, e2e-mol.3, e2e-mac.1, e2e-t-ro] + - [comments, add, e2e-r1, "seed comment"] + +shapes: + # ---------------------------------------------------------------- show + - id: gc-show-json + source: gc bdstore.go BdStore.Get (show --json ); counts must not read 0 + args: [show, --json, e2e-blk] + - id: gc-show-json-batch + source: gc bdstore_exact_batch.go GetExactBatch + args: [show, --json, e2e-r1, e2e-a1, e2e-blk, e2e-mol] + - id: gc-show-json-wisp + source: gc workquery.go blocked_by enrichment; wisps live in the ready tier + args: [show, e2e-wisp-1, --json] + - id: gc-show-id-flag + source: gc cmd_bd_show_watch.go bdWatchRunner (show --json --id=) + args: [show, --json, --id=e2e-r1] + # The worker holds eight in-progress issues; e2e-ip1 is the only P0 among + # them, so "first in the store's default order" (priority first) names it on + # both sides. Among equal priorities the order rests on created_at, and a + # seed that straddles a second boundary on one side only picked another issue. + - id: gc-show-current + source: gc cmd_bd_show_watch.go bdWatchRunner (show --current --json) + args: [show, --current, --json] + pending: {slice: "S6e", reason: "NEW: resolveCurrentIssueID calls raw store.SearchIssues, the http refusal is swallowed, and it silently falls back to the last-touched issue (wrong issue, exit 0)"} + - id: gc-show-missing + source: gc BdStore.Get not-found classification (routing fallbacks) + args: [show, --json, e2e-nonexistent] + expect_exit: 1 + expect_stderr: "(?i)not found" + compare: exit + - id: mc-show-text + source: mc prompts (bd show , ~80) + args: [show, e2e-r1] + compare: text + + # ---------------------------------------------------------------- ready + # e2e-wr and e2e-wisp-wr are the only open rows assigned to the worker when + # these shapes run, so the assignee shapes compare real rows: a client that + # sends the wrong assignee or drops --include-ephemeral returns a different + # list. The wisp is P1 and the task P2, so --limit=1 picks the wisp on both + # sides by priority alone. + - id: gc-ready-unlimited + source: gc bdstore.go bdReadyArgs (BdStore.Ready, ready --json --limit 0) + args: [ready, --json, --limit, "0"] + compare: ids + pending: {slice: "S15", reason: "ready --limit 0 omits limit and the server default page silently caps the answer (B3 #2)"} + - id: gc-ready-ephemeral-assignee-unlimited + source: gc bdstore.go bdReadyArgs with --include-ephemeral --assignee + args: [ready, --json, --include-ephemeral, --assignee, worker, --limit, "0"] + compare: ids + - id: gc-ready-pool-demand + source: gc workquery.go bdReadyPoolDemandShell (scale_check, --limit=20) + args: [ready, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit=20] + compare: ids + - id: gc-ready-pool-demand-unlimited + source: gc workquery.go bdReadyPoolDemandShell (work_query, --include-ephemeral --limit 0) + args: [ready, --include-ephemeral, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit, "0"] + compare: ids + pending: {slice: "S15", reason: "ready --limit 0 silently capped by the server's default page (B3 #2)"} + - id: gc-ready-workflow-migration + source: gc workquery.go bdReadyPoolDemandMigrationShell (--sort oldest --limit 0) + args: [ready, --include-ephemeral, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit, "0"] + compare: ids + - id: gc-ready-graph-anchor + source: gc workquery.go assignedGraphWorkflowAnchorReadyFunctionScript + args: [ready, --include-ephemeral, --metadata-field, gc.root_bead_id=e2e-mol, --metadata-field, gc.routed_to=pool-a, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit=20] + compare: ids + - id: gc-ready-assigned-tier + source: gc workquery.go assignedReadyTierCommand (--assignee= --limit=1) + args: [ready, --include-ephemeral, --assignee=worker, --json, --limit=1] + compare: ids + - id: gc-ready-probe + source: gc examples/bd gc-beads-bd.sh (bare bd ready probe) + args: [ready] + compare: exit + - id: mc-ready-json + source: mc scripts/prompts (bd ready --json, ~36) + args: [ready, --json] + compare: ids + - id: mc-ready-mol + source: mc skills/pr-review (bd ready --mol ) + args: [ready, --mol, e2e-mol, --json] + pending: {slice: "S16", reason: "ready --mol loads molecule steps off a raw path: step rows lose assignee and timestamps (molecule reads over roles)"} + - id: mc-ready-parent + source: mc skills/pr-review (bd ready --parent ) + args: [ready, --parent, e2e-ep, --json] + compare: ids + + # ---------------------------------------------------------------- list + - id: gc-list-full-filters + source: gc bdstore.go listViaBDList (List/ListByLabel/ListByMetadata) + args: [list, --json, --label=lane:a, --status=open, --type=task, --include-infra, --include-gates, --limit, "0", --metadata-field, gc.routed_to=pool-a] + - id: gc-list-ping + source: gc bdstore.go BdStore.Ping (list --json --limit 0) + args: [list, --json, --limit, "0"] + compare: ids + - id: gc-list-limit1 + source: gc doctor_gate_sandbox.go / doctor_store_preflight.go + args: [list, --json, --limit, "1"] + compare: count + - id: gc-list-readonly + source: gc doctor_v2_session_migration.go readOnlyBdRunner + args: [--readonly, list, --json, --limit, "1"] + compare: count + - id: gc-list-assignee-status + source: gc bdstore.go listViaBDList (ListByAssignee) + args: [list, --json, --assignee=worker, --status=in_progress, --include-infra, --include-gates, --limit, "0"] + - id: gc-list-all-created-before + source: gc bdstore.go listViaBDList (--all --created-before --include-templates --skip-labels) + args: [list, --json, --all, --created-before, "2100-01-01T00:00:00Z", --include-infra, --include-gates, --include-templates, --limit, "0", --skip-labels] + compare: ids + - id: gc-list-children + source: gc bdstore.go listViaBDList (Children, --parent) + args: [list, --json, --include-infra, --include-gates, --limit, "0", --parent, e2e-ep, --all] + - id: gc-list-in-progress-tier + source: gc workquery.go assignedInProgressTierCommandWithLimit + args: [list, --status, in_progress, --assignee=worker, --json, --limit=20] + - id: gc-list-on-death + source: gc workquery.go EffectiveOnDeath + args: [list, --assignee=worker, --status=in_progress, --json] + - id: gc-list-on-boot + source: gc workquery.go on_boot recovery (--no-assignee) + args: [list, --metadata-field, gc.routed_to=pool-a, --status=in_progress, --no-assignee, --json] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + - id: gc-list-on-boot-workflow + source: gc workquery.go on_boot recovery, workflow lane + args: [list, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --status=in_progress, --no-assignee, --json] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + - id: mc-list-meta-status + source: mc assets snapshot.md (list --json --limit --metadata-field --status) + args: [list, --json, --limit, "50", --metadata-field, gc.routed_to=pool-a, --status, open] + compare: ids + # More than 50 rows match, most at one priority: --limit 50 cuts a tie set + # whose created_at order differs between the two seeds, so each side's 50 + # rows are held to its own unlimited listing instead. + prefix_of: [list, --json, --limit, "0", --metadata-field, gc.routed_to=pool-a, --status, open] + - id: mc-list-github-source + source: mc github-intake (list --all --json --limit --metadata-field x3) + args: [list, --metadata-field, gc.kind=github_source, --metadata-field, gc.github.kind=issue, --metadata-field, gc.github.repo=o/r, --all, --json, --limit, "0"] + - id: mc-list-all-type + source: mc normalized_candidate_graph.py (list --all --type) + args: [list, --all, --type, epic, --json, --limit, "50"] + - id: mc-list-include-gates-infra-label + source: mc mol-normalized-issue-pr-v1 (list --all --include-gates --include-infra --json --label --limit) + args: [list, --all, --include-gates, --include-infra, --json, --label, "lane:a", --limit, "0"] + - id: mc-list-title + source: mc mol-normalized-issue-pr-v1 (list --all --include-infra --json --limit --title) + args: [list, --all, --include-infra, --json, --limit, "10", --title, "Routed open task"] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --title is a text filter the list encoder refuses (B3 #11)"} + - id: mc-list-id + source: mc skills/pr-review (list --all --id --json --limit) + args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] + pending: {slice: "S6c+S15+S15b", reason: "list stub (B3 #1); --limit 0 (B3 #3); --id refused by the list encoder (B3 #11)"} + - id: mc-list-assignee-status + source: mc review summaries (list --assignee --json --limit --status) + args: [list, --assignee, alice, --json, --limit, "10", --status, open] + - id: mc-list-label-text + source: mc normalized_child_launch.py (list --label) + args: [list, --label, "lane:a"] + compare: text + - id: mc-list-plain + source: mc scripts (bare bd list) + args: [list] + compare: exit + - id: mc-children + source: mc prompts (bd children ) + args: [children, e2e-ep, --json] + + # ---------------------------------------------------------------- query / count / search + - id: gc-query-wisps + source: gc bdstore.go listEphemeral (query --json "ephemeral=true AND ..." --limit N) + args: [query, --json, "ephemeral=true AND label=lane:w", --limit, "50"] + - id: gc-query-wisps-all + source: gc bdstore.go listEphemeral with IncludeClosed (--all) + args: [query, --json, "ephemeral=true AND status=in_progress", --all, --limit, "50"] + - id: gc-query-wisp-by-id + source: gc bdstore.go getEphemeralByID + args: [query, --json, "ephemeral=true AND id=e2e-wisp-1", --all, --limit, "1"] + - id: gc-query-wisps-unlimited + source: gc workquery.go bdQueryEphemeralStatusShell (--limit=0, wisp tier) + args: [query, --json, "ephemeral=true AND status=open", --limit=0] + compare: ids + pending: {slice: "S15", reason: "query --limit=0 sends limit=0 and a non-loopback server refuses unlimited reads (B3 #3)"} + - id: mc-query + source: mc pr_merge.py (bd query) + args: [query, --json, "status=open AND priority<=1", --limit, "50"] + - id: gc-count + source: gc native CountIssues fallback (count without scope) + args: [count, --json] + - id: gc-count-status + source: gc count with a status filter + args: [count, --json, --status, open] + - id: mc-search + source: mc mol-bug-report-flow-v2 (bd search) + args: [search, "Routed", --json] + compare: ids + pending: {slice: "S24", reason: "NEW: bd search is refused over http ('partial-id search is not available over HTTP'); not in B3"} + - id: mc-status + source: mc normalized_child_launch.py (bd status) + args: [status, --json] + + # ---------------------------------------------------------------- deps / graph reads + - id: gc-dep-list + source: gc bdstore.go DepList + args: [dep, list, e2e-blk, --json] + - id: gc-dep-list-up + source: gc bdstore.go DepList (--direction=up) + args: [dep, list, e2e-r1, --json, --direction=up] + - id: gc-dep-list-batch + source: gc bdstore.go DepListBatch + args: [dep, list, e2e-blk, e2e-mol.2, --json] + - id: mc-dep-tree + source: mc pr_review.py city_dep_subtree (dep tree --json) + args: [dep, tree, e2e-mol, --json] + - id: gc-blocked + source: gc bdstore_ready_projection.go fetchReadyProjectionViaBlocked + args: [blocked, --json] + pending: {slice: "S18", reason: "bd blocked hits the GetBlockedIssues stub (B3 #10)"} + + # ---------------------------------------------------------------- molecules + - id: mc-mol-current + source: mc worker prompts (bd mol current, ~38; inferred from in_progress work) + args: [mol, current, --json] + pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} + - id: mc-mol-current-text + source: mc gc-role-worker template (bd mol current, text) + args: [mol, current] + compare: text + pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} + - id: mc-mol-current-id + source: mc prompts (bd mol current ) + args: [mol, current, e2e-mol, --json] + pending: {slice: "S16", reason: "mol current refuses (B3 #4)"} + - id: mc-mol-progress + source: mc prompts (bd mol progress ) + args: [mol, progress, e2e-mol, --json] + pending: {slice: "S16", reason: "mol progress refuses (B3 #4)"} + + # ---------------------------------------------------------------- config / probes / local + - id: gc-config-get-types + source: gc beads_provider_lifecycle.go / checks_custom_types.go (config get --json types.custom) + args: [config, get, --json, types.custom] + - id: gc-config-get-prefix + source: gc gc-beads-bd.sh (config get issue_prefix) + args: [config, get, issue_prefix] + compare: text + - id: gc-types + source: gc checks_custom_types.go (types --json) + args: [types, --json] + - id: gc-context + source: gc beads_preflight_checker.go (context --json); never touches the store + args: [context, --json] + compare: exit + - id: gc-version + source: gc bdstore_blocked_repair.go BDVersion + args: [version] + compare: exit + - id: gc-ping + source: gc cmd_beads_city_migrate_proxied.go / dashboard (ping --json) + args: [ping, --json] + compare: exit + - id: mc-prime + source: mc formulas (bd prime) + args: [prime] + compare: exit + - id: gc-sql-ready-projection + source: gc bdstore_ready_projection.go fetchReadyProjection + args: [sql, "select id,is_blocked from issues where status <> 'closed' union all select id,is_blocked from wisps where status <> 'closed'", --json] + expect_exit: 1 + expect_stderr: "'bd sql' is not yet supported" + compare: exit + - id: gc-sql-release-fallback + source: gc bdstore.go ReleaseIfCurrent fallback (sql UPDATE) + args: [sql, --json, "UPDATE issues SET status='open', assignee='' WHERE id='e2e-t-rel' AND status='in_progress' AND assignee='worker'"] + expect_exit: 1 + expect_stderr: "'bd sql' is not yet supported" + compare: exit + - id: gc-purge-dry-run + source: gc bdstore.go Purge (purge --json --dry-run) + args: [purge, --json, --dry-run] + compare: exit + pending: {slice: "SRV-1", reason: "purge needs the sweep tokens the reference server does not advertise (B3 #13); remove the mark when reference_server.json is upgraded"} + - id: gc-recompute-blocked + source: gc bdstore_blocked_repair.go RecomputeBlocked + args: [recompute-blocked, --json] + compare: exit + pending: {slice: "S25", reason: "NEW: recompute-blocked refuses over http ('storage backend does not support is_blocked recompute'); gc BdStore.RecomputeBlocked issues it; not in B3"} + - id: gc-backup-status + source: gc checks_proxied_backup_coverage.go (backup status --json) + args: [backup, status, --json] + compare: exit + - id: mc-gate-list + source: core pack orders (gate list) + args: [gate, list, --json] + pending: {slice: "S20", reason: "gate list issues a raw SearchIssues shape the bridge refuses (B3 #15)"} + + - id: gc-export-all + source: gc core pack reaper (gc bd export --all -o F) + args: [export, --all, -o, "{file:export.jsonl}"] + compare: exit + pending: {slice: "S21", reason: "export --all is not wired over http (B3 #16)"} + + # ================================================================ writes + # ---------------------------------------------------------------- create + - id: gc-create-full + source: gc bdstore.go CreateWithStorage (every optional flag but --parent) + args: [create, --json, "gc created", -t, task, --id, e2e-n1, --priority, "1", --description, "made by gc", --assignee, worker, --deps, e2e-r1, --labels, "lane:n,x", --metadata, '{"gc.routed_to":"pool-a"}'] + compare: exit + verify: [[show, --json, e2e-n1]] + - id: gc-create-parent + source: gc bdstore.go CreateWithStorage (--parent, server-minted child id) + args: [create, --json, "gc child", -t, task, --parent, e2e-ep] + compare: exit + verify: [[show, --json, e2e-ep.3]] + pending: {slice: "S19", reason: "create --parent calls GetNextChildID (B3 #8)"} + - id: gc-create-ephemeral + source: gc bdstore.go CreateWithStorage (--ephemeral) + args: [create, --json, "gc wisp", -t, task, --id, e2e-wisp-n2, --ephemeral, --metadata, '{"gc.routed_to":"pool-a"}'] + compare: exit + verify: [[show, --json, e2e-wisp-n2]] + - id: gc-create-no-history + source: gc bdstore.go CreateWithStorage (--no-history, agent beads) + args: [create, --json, "gc agent bead", -t, task, --id, e2e-n3, --no-history] + compare: exit + verify: [[show, --json, e2e-n3]] + - id: gc-create-defer + source: gc bdstore.go CreateWithStorage (--defer RFC3339) + args: [create, --json, "gc deferred", -t, task, --id, e2e-n4, --defer, "2100-01-01T00:00:00Z"] + compare: exit + verify: [[show, --json, e2e-n4]] + - id: mc-create-external-ref + source: mc snapshot.md (create --external-ref --labels --metadata --type) + args: [create, "triaged issue", --external-ref, gh-999, --labels, "triage", --metadata, '{"gc.kind":"github_source"}', --type, task, --id, e2e-n5, --json] + compare: exit + verify: [[show, --json, e2e-n5]] + - id: mc-create-text + source: mc prompts (bd create, text output) + args: [create, "plain create", --type, task, --id, e2e-n6] + compare: exit + verify: [[show, --json, e2e-n6]] + - id: gc-create-graph + source: gc bdstore_graph_apply.go ApplyGraphPlanWithStorage + args: [create, --graph, "{file:graph-small.json}", --json] + verify: [[show, --json, e2e-g-root, e2e-g-s1, e2e-g-s2]] + - id: gc-create-graph-ephemeral + source: gc bdstore_graph_apply.go (--ephemeral by storage class) + args: [create, --graph, "{file:graph-eph.json}", --json, --ephemeral] + verify: [[show, --json, e2e-wisp-g-root, e2e-wisp-g-s1]] + + # ---------------------------------------------------------------- update + - id: gc-update-fields + source: gc bdstore.go bdUpdateArgs (Update) + args: [update, --json, e2e-t-upd, --title, "updated", --status, in_progress, --type, bug, --priority, "1", --description, "new", --assignee, bob, --set-metadata, a=1, --set-metadata, b=2, --add-label, x, --remove-label, "lane:z"] + compare: exit + verify: [[show, --json, e2e-t-upd]] + - id: gc-update-parent + source: gc bdstore.go bdUpdateArgs (--parent) + args: [update, --json, e2e-t-upd2, --parent, e2e-ep] + compare: exit + verify: [[show, --json, e2e-t-upd2]] + - id: gc-update-many + source: gc bdstore.go UpdateAll / setMetadataBatchAll + args: [update, --json, e2e-t-m1, e2e-t-m2, --set-metadata, gc.batch=1, --priority, "3"] + compare: exit + verify: [[show, --json, e2e-t-m1, e2e-t-m2]] + - id: gc-update-claim + source: gc bdstore.go BdStore.Claim and gc hook --claim (update --claim --json) + args: [update, e2e-t-claim, --claim, --json] + compare: exit + verify: [[show, --json, e2e-t-claim]] + - id: gc-update-claim-actor + source: gc cmd_agent_script.go bd_claim (update --claim --actor X) + args: [update, e2e-t-claim2, --claim, --actor, alice] + compare: exit + verify: [[show, --json, e2e-t-claim2]] + - id: gc-update-claim-wisp + source: gc ready queues include ephemeral work; claiming a wisp + args: [update, e2e-wisp-c, --claim, --json] + compare: exit + verify: [[show, --json, e2e-wisp-c]] + pending: {slice: "SRV-1", reason: "the reference server lacks UpdateIssueRequest.claim, and the claimIssue fallback excludes wisps (B3 #6); remove the mark when reference_server.json is upgraded"} + - id: gc-update-if-revision + source: gc bdstore_conditional.go UpdateIfMatch / CompareAndSetMetadataKey + args: [update, e2e-t-rev, --set-metadata, k=v, --if-revision, "{rev:e2e-t-rev}"] + compare: exit + verify: [[show, --json, e2e-t-rev]] + - id: gc-update-if-revision-stale + source: gc bdstore_conditional.go UpdateIfMatch mismatch (exit 13) + args: [update, e2e-t-rev2, --set-metadata, k=v, --if-revision, "1"] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-rev2]] + - id: gc-update-release-if-current + source: gc bdstore_conditional_release.go releaseIfCurrentViaBdVerb + args: [update, e2e-t-rel, --if-assignee, worker, --if-status, in_progress, --status, open, --assignee, ""] + compare: exit + verify: [[show, --json, e2e-t-rel]] + - id: gc-update-transfer-if-current + source: gc bdstore_conditional_release.go TransferIfCurrent + args: [update, e2e-t-xfer, --if-assignee, worker, --if-status, in_progress, --assignee, alice] + compare: exit + verify: [[show, --json, e2e-t-xfer]] + - id: gc-update-if-assignment + source: gc bdstore_conditional.go UpdateIfAssignment + args: [update, --json, e2e-t-ifa, --set-metadata, k=v, --if-status, in_progress, --if-assignee, worker] + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-if-assignment-assignee-stale + source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) + args: [update, --json, e2e-t-ifa, --set-metadata, probe=a, --if-status, in_progress, --if-assignee, nobody] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-if-assignment-status-stale + source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) + args: [update, --json, e2e-t-ifa, --set-metadata, probe=s, --if-status, open, --if-assignee, worker] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-on-death + source: gc workquery.go on_death (update --assignee "" --status open --set-metadata) + args: [update, e2e-t-death, --assignee, "", --status, open, --set-metadata, gc.run_target=pool-a] + compare: exit + verify: [[show, --json, e2e-t-death]] + - id: gc-update-on-boot + source: gc workquery.go on_boot (update --status open) + args: [update, e2e-t-boot, --status, open] + compare: exit + verify: [[show, --json, e2e-t-boot]] + - id: gc-update-sling + source: gc workquery.go DefaultSlingQuery (update --set-metadata gc.routed_to=) + args: [update, e2e-t-sling, --set-metadata, gc.routed_to=pool-b] + compare: exit + verify: [[show, --json, e2e-t-sling]] + - id: gc-update-agent-script + source: gc cmd_agent_script.go bd_update (--assignee --status --append-notes --set-metadata) + args: [update, e2e-t-as, --assignee, worker, --status, in_progress, --append-notes, "agent note", --set-metadata, k=v] + compare: exit + verify: [[show, --json, e2e-t-as]] + - id: gc-update-dolt-auto-commit-off + source: gc bdstore.go bdTransientWriteArgs (--dolt-auto-commit off prefix) + args: [--dolt-auto-commit, "off", update, --json, e2e-t-dac, --set-metadata, k=v] + compare: exit + verify: [[show, --json, e2e-t-dac]] + - id: mc-update-meta-status + source: mc prompts (update --set-metadata --status, ~151) + args: [update, e2e-t-ms, --set-metadata, review.verdict=iterate, --status, in_progress] + compare: exit + verify: [[show, --json, e2e-t-ms]] + - id: mc-update-meta-status-closed + source: mc prompts (update --set-metadata --status closed, under Claude Code) + args: [update, e2e-t-msc, --set-metadata, review.verdict=done, --status, closed] + compare: exit + verify: [[show, --json, e2e-t-msc]] + pending: {slice: "S6d", reason: "a silent data difference, not a refusal: the update succeeds on both sides under CLAUDE_SESSION_ID, but the served update records no closed_by_session (an http update patch cannot carry one, so the CLI drops it without a word; bd close does carry it) where embedded records the session, so the verify show differs (B3 #7)"} + - id: mc-update-unset-metadata + source: mc adopt-pr-operator (update --unset-metadata) + args: [update, e2e-t-um, --unset-metadata, k] + compare: exit + verify: [[show, --json, e2e-t-um]] + - id: mc-update-notes + source: mc github fix-issue formula (update --notes) + args: [update, e2e-t-notes, --notes, "notes text"] + compare: exit + verify: [[show, --json, e2e-t-notes]] + - id: mc-update-external-ref-metadata + source: mc snapshot.md (update --external-ref --metadata) + args: [update, e2e-t-er, --external-ref, gh-1, --metadata, '{"gc.kind":"github_source"}'] + compare: exit + verify: [[show, --json, e2e-t-er]] + - id: mc-update-metadata + source: mc build-base prepare.md (update --metadata) + args: [update, e2e-t-md, --metadata, '{"x":"1"}'] + compare: exit + verify: [[show, --json, e2e-t-md]] + - id: mc-update-assignee + source: mc review reports (update --assignee) + args: [update, e2e-t-asg, --assignee, bob] + compare: exit + verify: [[show, --json, e2e-t-asg]] + + # ---------------------------------------------------------------- close / reopen + - id: gc-close-force-reason-many + source: gc bdstore.go bdCloseArgs (CloseAll/CloseAllWithReason) + args: [close, --force, --json, --reason, done, e2e-t-c1, e2e-t-c2] + compare: exit + verify: [[show, --json, e2e-t-c1, e2e-t-c2]] + - id: gc-close-if-revision + source: gc bdstore_conditional.go CloseIfMatch + args: [close, --force, --json, e2e-t-c3, --if-revision, "{rev:e2e-t-c3}"] + compare: exit + verify: [[show, --json, e2e-t-c3]] + - id: mc-close-text + source: mc formulas (bd close ) + args: [close, e2e-t-c4] + compare: exit + verify: [[show, --json, e2e-t-c4]] + - id: mc-close-reason + source: mc build-basic-review (bd close --reason) + args: [close, e2e-t-c5, --reason, "review applied"] + compare: exit + verify: [[show, --json, e2e-t-c5]] + - id: gc-close-last-step-autocloses-molecule + source: gc/mc molecule lifecycle (closing the last open step closes the root) + args: [close, --force, --json, e2e-mac.2] + compare: exit + verify: [[show, --json, e2e-mac]] + - id: gc-reopen + source: gc bdstore.go Reopen + args: [reopen, --json, e2e-t-ro] + compare: exit + verify: [[show, --json, e2e-t-ro]] + + # ---------------------------------------------------------------- delete / reclaim / promote / burn + - id: gc-delete + source: gc bdstore.go Delete + args: [delete, --force, --json, e2e-t-d1] + compare: exit + verify: [[show, --json, e2e-t-d1]] + - id: gc-delete-if-revision + source: gc bdstore_conditional.go DeleteIfMatch + args: [delete, --force, --json, e2e-t-d2, --if-revision, "{rev:e2e-t-d2}"] + compare: exit + verify: [[show, --json, e2e-t-d2]] + - id: gc-delete-batch + source: gc bdstore.go DeleteBatch (no --json) + args: [delete, e2e-t-d3, e2e-t-d4, --force] + compare: exit + verify: [[show, --json, e2e-t-d3]] + - id: gc-delete-cascade + source: gc cmd_convoy_dispatch.go deleteWorkflowMatches + args: [delete, e2e-t-dc, --cascade, --force] + compare: exit + verify: [[show, --json, e2e-t-dc.1]] + - id: mc-delete-from-file + source: mc session_cleanup.py / core reaper (delete --from-file) + args: [delete, --from-file, "{file:delete-list.txt}", --force] + compare: exit + verify: [[show, --json, e2e-t-df1]] + - id: gc-reclaim + source: gc bdstore.go ReclaimStale (reclaim --id --json) + args: [reclaim, --id, e2e-t-rc, --json] + compare: exit + verify: [[show, --json, e2e-t-rc]] + pending: {slice: "SRV-1", reason: "the reference server does not advertise issues.reclaim (B3 #12), so the client refuses at its capability preflight"} + - id: mc-promote + source: mc mol-normalized-issue-pr-v1 (promote --reason) + args: [promote, e2e-wisp-p, --reason, "keep it"] + compare: exit + pending: {slice: "S21", reason: "promote is not wired over http (B3 #16)"} + - id: mc-mol-burn + source: mc review fixtures (mol burn --force) + args: [mol, burn, e2e-wisp-m, --force] + compare: exit + verify: [[show, --json, e2e-wisp-m.1]] + pending: {slice: "S17", reason: "mol burn needs RunInTransaction (B3 #14)"} + + # ---------------------------------------------------------------- deps / comments / labels / config writes + - id: gc-dep-add + source: gc bdstore.go DepAdd + args: [dep, add, e2e-t-da, e2e-t-db, --type, blocks] + compare: exit + verify: [[dep, list, e2e-t-da, --json]] + - id: gc-dep-remove + source: gc bdstore.go DepRemove + args: [dep, remove, e2e-t-dr, e2e-t-db] + compare: exit + verify: [[dep, list, e2e-t-dr, --json]] + - id: mc-comment + source: mc bug_hunt.py (bd comment ) + args: [comment, e2e-t-cm, "a comment"] + compare: exit + verify: [[comments, e2e-t-cm, --json]] + ignore_fields: [id] # comment ids are minted per workspace + - id: mc-comments-list + source: mc prompts (bd comments ) + args: [comments, e2e-r1, --json] + ignore_fields: [id] # comment ids are minted per workspace + - id: mc-label-add + source: mc prompts (bd label add) + args: [label, add, e2e-t-l, "lane:q"] + compare: exit + verify: [[show, --json, e2e-t-l]] + - id: mc-assign + source: mc prompts (bd assign) + args: [assign, e2e-t-as2, alice] + compare: exit + verify: [[show, --json, e2e-t-as2]] + - id: gc-config-set-types + source: gc beads_provider_lifecycle.go (config set types.custom a,b) + args: [config, set, types.custom, "corpus-a,corpus-b"] + compare: exit + verify: [[config, get, --json, types.custom]] + - id: mc-meta-set + source: mc adopt-pr-review checks (bd meta set k=v) + args: [meta, set, e2e-r1, review.verdict=done] + expect_exit: 1 + expect_stderr: 'unknown command "meta"' + compare: exit diff --git a/backend/http/corpus/testdata/reference_server.json b/backend/http/corpus/testdata/reference_server.json new file mode 100644 index 000000000..7bdf6d0d0 --- /dev/null +++ b/backend/http/corpus/testdata/reference_server.json @@ -0,0 +1,595 @@ +{ + "comment": [ + "The reference server profile the corpus suite masks the in-process server to.", + "It is the wire surface of a deployed bd 1.3.0 `bd serve` that predates wire_revision:", + "capabilities is that server's advertised ContextResponse.capabilities (read off a live GET /v0/beads/context),", + "context_members is its ContextResponse property set, and operations are the method, path template,", + "query parameter names and top-level JSON request members of its published OpenAPI document (MIT).", + "Regenerate by hand when the reference deployment is upgraded; see README.md." + ], + "capabilities": [ + "config.get", + "config.list", + "config.set", + "config.unset", + "dependencies.add", + "dependencies.blocking", + "dependencies.count", + "dependencies.cycles", + "dependencies.list", + "dependencies.remove", + "dependencies.tree", + "events.list", + "events.watch", + "issues.addComment", + "issues.batchApply", + "issues.batchClose", + "issues.batchCreate", + "issues.casMetadata", + "issues.claim", + "issues.claimNext", + "issues.close", + "issues.count", + "issues.create", + "issues.delete", + "issues.get", + "issues.list", + "issues.list.sort", + "issues.query", + "issues.related", + "issues.release", + "issues.reopen", + "issues.sweep", + "issues.update", + "memories.forget", + "memories.get", + "memories.list", + "memories.remember", + "policy.external_dependencies", + "project.enforce", + "ready.count", + "ready.list", + "stats.get" + ], + "context_members": [ + "api_version", + "backend", + "bd_version", + "beads_dir", + "capabilities", + "database", + "dolt_mode", + "project_id", + "repo_root", + "schema_version" + ], + "operations": [ + { + "method": "GET", + "path": "/healthz", + "operation_id": "health", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/config", + "operation_id": "listSettings", + "query": [], + "body": [] + }, + { + "method": "DELETE", + "path": "/v0/beads/config/{key}", + "operation_id": "unsetSetting", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/config/{key}", + "operation_id": "getSetting", + "query": [], + "body": [] + }, + { + "method": "PUT", + "path": "/v0/beads/config/{key}", + "operation_id": "setSetting", + "query": [], + "body": [ + "value" + ] + }, + { + "method": "GET", + "path": "/v0/beads/context", + "operation_id": "getContext", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies", + "operation_id": "listDependencies", + "query": [ + "issue_id", + "type" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/blocking", + "operation_id": "listBlockingAnnotations", + "query": [ + "issue_id" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/cycles", + "operation_id": "listDependencyCycles", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/tree", + "operation_id": "getDependencyTree", + "query": [ + "direction", + "max_depth", + "root_id", + "status" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/dependencies:add", + "operation_id": "addDependencies", + "query": [], + "body": [ + "actor", + "edges" + ] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies:count", + "operation_id": "countDependencyEdges", + "query": [ + "direction", + "issue_id", + "status", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/dependencies:remove", + "operation_id": "removeDependency", + "query": [], + "body": [ + "actor", + "depends_on_id", + "issue_id" + ] + }, + { + "method": "GET", + "path": "/v0/beads/events", + "operation_id": "listEvents", + "query": [ + "limit", + "since" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/events:watch", + "operation_id": "watchEvents", + "query": [ + "since" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/issues", + "operation_id": "listIssues", + "query": [ + "all", + "assignee", + "brief", + "created_after", + "created_before", + "cursor", + "exclude_label", + "has_metadata_key", + "include_ephemeral", + "include_gates", + "include_infra", + "include_templates", + "label", + "label_any", + "limit", + "metadata_field", + "parent", + "reverse", + "sort", + "status", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues", + "operation_id": "createIssue", + "query": [], + "body": [ + "acceptance_criteria", + "actor", + "assignee", + "defer_until", + "dependencies", + "description", + "design", + "due_at", + "ephemeral", + "estimated_minutes", + "external_ref", + "force_id_prefix", + "id", + "inherit_labels_from_parent", + "issue_type", + "labels", + "metadata", + "no_history", + "notes", + "owner", + "parent_id", + "priority", + "sender", + "status", + "title", + "waits_for" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues/{id}", + "operation_id": "getIssue", + "query": [ + "brief_deps", + "include_comments", + "include_dependents" + ], + "body": [] + }, + { + "method": "PATCH", + "path": "/v0/beads/issues/{id}", + "operation_id": "updateIssue", + "query": [], + "body": [ + "actor", + "expected_assignee", + "expected_status", + "expected_version", + "force_assignee_transfer", + "force_close_policy", + "patch" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}/comments", + "operation_id": "addComment", + "query": [], + "body": [ + "author", + "text" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues/{id}/related", + "operation_id": "listRelatedIssues", + "query": [ + "direction", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:casMetadata", + "operation_id": "compareAndSetMetadata", + "query": [], + "body": [ + "actor", + "expected", + "key", + "value" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:claim", + "operation_id": "claimIssue", + "query": [], + "body": [ + "actor" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:close", + "operation_id": "closeIssue", + "query": [], + "body": [ + "actor", + "expected_version", + "force", + "reason", + "session" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:release", + "operation_id": "releaseIssue", + "query": [], + "body": [ + "actor", + "expected_assignee", + "force" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:reopen", + "operation_id": "reopenIssue", + "query": [], + "body": [ + "actor", + "expected_version", + "reason" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchApply", + "operation_id": "applyBatch", + "query": [], + "body": [ + "actor", + "force_id_prefix", + "items", + "provenance", + "skip_per_edge_cycle_check" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchClose", + "operation_id": "batchCloseIssues", + "query": [], + "body": [ + "actor", + "claim_next", + "force", + "items", + "session" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchCreate", + "operation_id": "batchCreateIssues", + "query": [], + "body": [ + "actor", + "items" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:claimNext", + "operation_id": "claimNextIssue", + "query": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "body": [ + "actor" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues:count", + "operation_id": "countIssues", + "query": [ + "assignee", + "closed_after", + "closed_before", + "created_after", + "created_before", + "desc_contains", + "empty_description", + "group_by", + "id", + "include_infra", + "label", + "label_any", + "metadata_field", + "no_assignee", + "no_labels", + "notes_contains", + "priority", + "priority_max", + "priority_min", + "status", + "title", + "title_contains", + "type", + "updated_after", + "updated_before" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues:delete", + "operation_id": "deleteIssues", + "query": [], + "body": [ + "actor", + "cascade", + "dry_run", + "expected_version", + "force", + "ids" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues:query", + "operation_id": "queryIssues", + "query": [ + "all", + "limit", + "q", + "reverse", + "sort" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues:sweep", + "operation_id": "sweepIssues", + "query": [], + "body": [ + "actor", + "closed_before", + "dry_run", + "pattern", + "protect_referenced", + "tier" + ] + }, + { + "method": "GET", + "path": "/v0/beads/memories", + "operation_id": "listMemories", + "query": [ + "search" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/memories", + "operation_id": "rememberMemory", + "query": [], + "body": [ + "content", + "key" + ] + }, + { + "method": "DELETE", + "path": "/v0/beads/memories/{key}", + "operation_id": "forgetMemory", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/memories/{key}", + "operation_id": "getMemory", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/ready", + "operation_id": "listReadyWork", + "query": [ + "assignee", + "brief", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "limit", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/ready:count", + "operation_id": "countReadyWork", + "query": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "type", + "unassigned" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/stats", + "operation_id": "getStats", + "query": [ + "assignee", + "skip_blocked" + ], + "body": [] + } + ] +} \ No newline at end of file diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 67f4cb9ed..24364853d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -516,6 +516,7 @@ go_binary( race = "off", visibility = [ "//backend/http:__pkg__", + "//backend/http/corpus:__pkg__", "//cmd/bd:__subpackages__", "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 3b4178066..595887d92 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -41,7 +41,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | -| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | +| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 41cab234f..31dd9fa3c 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -907,6 +907,12 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + // The served command corpus (backend/http/corpus/README.md), with the + // same switches as http_served_test. + "//backend/http/corpus:corpus_served_test": { + []string{"$(rootpath :corpus_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{ From 7477ec1352d9380b692773bc3c2e66ae2e60ae37 Mon Sep 17 00:00:00 2001 From: "gascityinc-randy[bot]" <279382806+gascityinc-randy[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:58:58 +0000 Subject: [PATCH 05/99] fix(deps): bump grpc to 1.83.2 and thrift to 0.24.0 (#7495) Resolve Dependabot alerts #146 (apache/thrift < 0.24.0) and #151 (google.golang.org/grpc <= 1.83.1) on the root module with the minimal bump: no other module moves, and both need only go 1.25, so the Nix flake stays on its current nixpkgs. Recompute default.nix vendorHash for the new go.sum. This lands the security fix independently of the Dolt 2.4.1 upgrade in #7441, which carries the same two bumps. Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5.5 (1M context) --- default.nix | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/default.nix b/default.nix index 8f5fdadf4..22a4bc8bf 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-9OqMZK3wL838RGOXuOsKZvcyqk43tCmv75EUo+iqSVQ="; + vendorHash = "sha256-IL80Qs9c7ZG17R/hRF6D+F6WVmGB0I+k60+nFuUpIJA="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index 8c47b11c6..96264246b 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect - github.com/apache/thrift v0.23.0 // indirect + github.com/apache/thrift v0.24.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect @@ -273,7 +273,7 @@ require ( google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect - google.golang.org/grpc v1.83.1 // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index d3dd784cc..9489f5919 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6IC github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= -github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -1720,8 +1720,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From bf7c19ccc465f587e130ae3571b7c97cfe8e9c2d Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Mon, 12 Oct 2026 05:37:54 +0000 Subject: [PATCH 06/99] fix(wisp): bd mol wisp gc --age 0s no longer misses a just-created wisp (#7246) * fix(wisp): --age 0s treats every wisp as abandoned, even one Dolt rounded into the future (#7236) findAbandonedWisps compared now.Sub(issue.UpdatedAt) > ageThreshold unconditionally. Dolt's DATETIME(0) column rounds fractional seconds half-up on write rather than truncating: a wisp created in the second half of a wall-clock second can read back with updated_at up to 0.5s in the future, making the comparison negative and failing the check for a wisp that is, by definition, not yet one second old. --age 0s's own documented contract ("hasn't been updated in --age duration") means every wisp qualifies trivially -- nothing has been updated in the last zero seconds. Extracted the comparison into wispIsAbandoned(now, updatedAt, ageThreshold), which short-circuits true whenever ageThreshold <= 0, bypassing the clock comparison (and its rounding exposure) entirely for that case. New TestWispIsAbandoned is a pure, deterministic unit test -- the original TestCLI_WispGCExcludeType needs a live Dolt test container and the reporter's own 60 real runs against one could not reliably reproduce the race, so a timing-dependent regression test would be exactly as unreliable. Confirmed red against the old inline comparison (the two age-0s cases failed) before extracting the fix, green after. go build/vet/gofmt clean. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0185qSpS74AbkmHoKRPf2yDD * chore: empty commit to retrigger mergeability recompute GitHub's mergeStateStatus has shown DIRTY/CONFLICTING since 2026-10-06, but a real `git merge --no-commit --no-ff` against current upstream/main auto-merges cleanly on all 3 touched files -- no actual conflict. This commit forces GitHub to recompute. Co-Authored-By: Claude Sonnet 5 * test(wisp): pin the gc call site and drop the 1ms age from the embedded test Folds the tests from #7472 into the #7236 fix. TestWispIsAbandoned pins the predicate. It cannot see whether findAbandonedWisps still decides age through it. TestFindAbandonedWisps AgesAtStoragePrecision drives findAbandonedWisps through a new wispGCNow clock seam with a wisp stored rounded into the future (written 12:00:00.7, stored 12:00:01): - --age 0s at 12:00:00.9 reclaims it. This fails against the old inline now.Sub(updated_at) > threshold. - --age 1h leaves it alone at that instant and reclaims it two hours later. - --age 1s at 12:00:01.5 leaves it alone: a positive threshold does not credit idle time that has not passed. TestWispGCProtectsActiveWisps ran gc with --age 1ms and relied on creation order to keep the most recently touched reclaimable wisp out of the rounding window. --age 0s states the intent ("every wisp is old enough") and is immune to the half-second rounding, so the ordering caveat goes. Co-authored-by: Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer Co-authored-by: Julian Knutsen --- CHANGELOG.md | 8 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/wisp.go | 28 ++++- cmd/bd/wisp_abandoned_test.go | 138 +++++++++++++++++++++++++ cmd/bd/wisp_gc_active_embedded_test.go | 18 ++-- 5 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 cmd/bd/wisp_abandoned_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4cb8a05..986e0cb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -433,6 +433,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 role through (see *Added*). A missing id is a plain not-found after one `getIssue`, instead of a partial-id search refusal, and `bd update --claim` answers a not-found gc classifies. +- **`bd mol wisp gc --age 0s` can miss a just-created wisp.** Dolt's + `DATETIME(0)` column rounds fractional seconds half-up on write rather + than truncating, so a wisp created in the second half of a wall-clock + second can read back with `updated_at` up to 0.5s in the future. The GC's + age check (`now.Sub(updatedAt) > ageThreshold`) then compares against a + negative duration and fails, even though `--age 0s`'s own contract is + that every wisp qualifies. `ageThreshold <= 0` now bypasses the clock + comparison entirely (#7236). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 24364853d..6506f2bd3 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1267,6 +1267,7 @@ go_test( "where_cgo_test.go", "where_embedded_test.go", "where_test.go", + "wisp_abandoned_test.go", "wisp_gc_active_embedded_test.go", "wisp_gc_purge_closed_embedded_test.go", "wisp_proxied_integration_test.go", diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index a1b7ac27e..82d0ee530 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -822,6 +822,30 @@ func runWispGC(cmd *cobra.Command, args []string) error { return nil } +// wispIsAbandoned reports whether a wisp counts as abandoned for GC +// purposes: it has not been updated within ageThreshold of now. +// +// ageThreshold <= 0 (--age 0s and below) always reports true, bypassing +// the clock comparison entirely — this matches the GC's own documented +// contract ("hasn't been updated in --age duration"): nothing has been +// updated in the last zero seconds. It also matters for a reason the +// comparison alone does not handle: Dolt's DATETIME(0) column rounds +// fractional seconds half-up on write rather than truncating (#7236), so +// a wisp created in the second half of a wall-clock second can read back +// with updatedAt up to 0.5s in the future. now.Sub(updatedAt) is then +// negative, failing a strict > comparison for a wisp that is, by +// definition, not yet one second old — exactly what --age 0s's "every +// wisp qualifies" contract exists to avoid depending on in the first +// place. +func wispIsAbandoned(now, updatedAt time.Time, ageThreshold time.Duration) bool { + return ageThreshold <= 0 || now.Sub(updatedAt) > ageThreshold +} + +// wispGCNow is the clock seam for findAbandonedWisps' age check, swappable in +// tests. Both gc routes read it, so a test that swaps it must not call +// t.Parallel(). +var wispGCNow = time.Now + func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThreshold time.Duration, excludeTypes []types.IssueType) ([]*types.Issue, error) { ephemeralFlag := true filter := types.IssueFilter{ @@ -848,7 +872,7 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre return nil, err } - now := time.Now() + now := wispGCNow() var abandoned []*types.Issue for _, issue := range issues { if r.IsInfraTypeCtx(ctx, issue.IssueType) { @@ -860,7 +884,7 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre if isProtectedWisp(issue, blockedSet, protectedStatuses) { continue } - if now.Sub(issue.UpdatedAt) > ageThreshold { + if wispIsAbandoned(now, issue.UpdatedAt, ageThreshold) { abandoned = append(abandoned, issue) } } diff --git a/cmd/bd/wisp_abandoned_test.go b/cmd/bd/wisp_abandoned_test.go new file mode 100644 index 000000000..e18c3397a --- /dev/null +++ b/cmd/bd/wisp_abandoned_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// TestWispIsAbandoned pins the #7236 fix: --age 0s (ageThreshold <= 0) must +// treat every wisp as abandoned regardless of its updatedAt, including a +// wisp whose stored updatedAt reads back AFTER now — the shape Dolt's +// DATETIME(0) half-up rounding on write produces for a wisp created in the +// second half of a wall-clock second (#7236). Before this fix, that case +// failed the strict now.Sub(updatedAt) > ageThreshold comparison because +// the difference is negative. +func TestWispIsAbandoned(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + + cases := []struct { + name string + updatedAt time.Time + age time.Duration + want bool + }{ + { + name: "age 0s: a wisp updated exactly now is abandoned", + updatedAt: now, + age: 0, + want: true, + }, + { + name: "age 0s: a wisp whose stored updatedAt rounded into the future is still abandoned", + updatedAt: now.Add(500 * time.Millisecond), + age: 0, + want: true, + }, + { + name: "a negative age behaves like zero", + updatedAt: now.Add(500 * time.Millisecond), + age: -time.Second, + want: true, + }, + { + name: "positive age: a fresh wisp is not abandoned", + updatedAt: now, + age: time.Hour, + want: false, + }, + { + name: "positive age: a wisp past the threshold is abandoned", + updatedAt: now.Add(-2 * time.Hour), + age: time.Hour, + want: true, + }, + { + name: "positive age: exactly at the threshold is not abandoned (strict >, unchanged)", + updatedAt: now.Add(-time.Hour), + age: time.Hour, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wispIsAbandoned(now, tc.updatedAt, tc.age); got != tc.want { + t.Errorf("wispIsAbandoned(%v, %v, %v) = %v, want %v", now, tc.updatedAt, tc.age, got, tc.want) + } + }) + } +} + +// fixedWispsMolReader serves findAbandonedWisps a fixed set of wisps with +// nothing blocked, no custom statuses and no dependents. It embeds the +// molReader INTERFACE, so a read the age scan does not make today panics +// rather than quietly answering zero. +type fixedWispsMolReader struct { + molReader + wisps []*types.Issue +} + +func (r fixedWispsMolReader) SearchIssues(context.Context, string, types.IssueFilter) ([]*types.Issue, error) { + return r.wisps, nil +} +func (fixedWispsMolReader) GetBlockedIssues(context.Context, types.WorkFilter) ([]*types.BlockedIssue, error) { + return nil, nil +} +func (fixedWispsMolReader) GetCustomStatusesDetailed(context.Context) ([]types.CustomStatus, error) { + return nil, nil +} +func (fixedWispsMolReader) IsInfraTypeCtx(context.Context, types.IssueType) bool { return false } +func (fixedWispsMolReader) FindWispDependentsRecursive(context.Context, []string) (map[string]bool, error) { + return nil, nil +} + +// TestFindAbandonedWispsAgesAtStoragePrecision pins the call site that +// TestWispIsAbandoned cannot see: findAbandonedWisps must decide age through +// wispIsAbandoned. A wisp written at 12:00:00.7 is stored as 12:00:01, and a +// gc at 12:00:00.9 with --age 0s must still reclaim it; the raw +// now.Sub(updated_at) > threshold that #7236 replaced reads -100ms and misses +// it. A one-hour threshold must leave the same wisp alone at that instant and +// reclaim it two hours later, so dropping the age check fails too. --age 1s +// at 12:00:01.5 must leave it alone: only half a second has passed since the +// stored second, so a positive threshold never credits idle time that has +// not elapsed. +func TestFindAbandonedWispsAgesAtStoragePrecision(t *testing.T) { + base := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) + wisp := &types.Issue{ + ID: "gc-idle", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Ephemeral: true, + UpdatedAt: base.Add(700 * time.Millisecond).Round(time.Second), // 12:00:01 + } + old := wispGCNow + t.Cleanup(func() { wispGCNow = old }) + + for _, tc := range []struct { + now time.Time + threshold time.Duration + want int + }{ + {base.Add(900 * time.Millisecond), 0, 1}, + {base.Add(900 * time.Millisecond), time.Hour, 0}, + {base.Add(2 * time.Hour), time.Hour, 1}, + {base.Add(1500 * time.Millisecond), time.Second, 0}, + } { + wispGCNow = func() time.Time { return tc.now } + got, err := findAbandonedWisps(context.Background(), fixedWispsMolReader{wisps: []*types.Issue{wisp}}, false, tc.threshold, nil) + if err != nil { + t.Fatalf("findAbandonedWisps(--age %s): %v", tc.threshold, err) + } + if len(got) != tc.want { + t.Errorf("findAbandonedWisps(--age %s) at %s, updated_at 12:00:01: got %d wisps, want %d", + tc.threshold, tc.now.Format("15:04:05.000"), len(got), tc.want) + } + } +} diff --git a/cmd/bd/wisp_gc_active_embedded_test.go b/cmd/bd/wisp_gc_active_embedded_test.go index 3d62d9f93..7ab85a319 100644 --- a/cmd/bd/wisp_gc_active_embedded_test.go +++ b/cmd/bd/wisp_gc_active_embedded_test.go @@ -28,14 +28,6 @@ func TestWispGCProtectsActiveWisps(t *testing.T) { // category. "reviewing" is wip (protected); "triaging" is active, which // behaves like plain open and stays reclaimable — that pair also guards // against the predicate degenerating into "protect everything". - // - // Ordering matters: every wisp expected to be RECLAIMED is created and - // last-touched first, before the protected ones. The age predicate is - // now.Sub(updated_at) > threshold against a timestamp written by the - // database, so the most recently touched bead can briefly appear - // not-yet-stale if the DB clock runs ahead of the test process. Protected - // wisps are excluded regardless of age, so only the reclaimable - // assertions are sensitive to this. bdCommand(t, bd, dir, "config", "set", "status.custom", "reviewing:wip,triaging:active") // Genuinely abandoned: an idle open ephemeral wisp. Must be reclaimed. @@ -66,10 +58,12 @@ func TestWispGCProtectsActiveWisps(t *testing.T) { customWIP := bdCreate(t, bd, dir, "custom wip step", "--ephemeral").ID bdCommand(t, bd, dir, "update", customWIP, "--status", "reviewing") - // Every wisp above was created well over 1ms ago by the time gc runs, so a - // 1ms threshold makes them all "stale" by updated_at. Only the genuinely - // abandoned ones should be reclaimed. - out := bdCommand(t, bd, dir, "mol", "wisp", "gc", "--age", "1ms", "--dry-run", "--json") + // --age 0s makes every wisp above old enough to reclaim. Any positive --age + // can leave the most recently touched one reading as too young, because + // updated_at is stored at whole-second precision and Dolt rounds the write + // up to half a second into the future (#7236). Protection is checked + // before age, so only the genuinely abandoned ones should be reclaimed. + out := bdCommand(t, bd, dir, "mol", "wisp", "gc", "--age", "0s", "--dry-run", "--json") var res struct { CleanedIDs []string `json:"cleaned_ids"` From b630f96f5bd231ff7aea78af0d47b6038f228d00 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 12 Oct 2026 09:02:22 +0000 Subject: [PATCH 07/99] fix(httpclient): unlimited ready/query/list reads over http, client-capped (S15) (#7497) * fix(httpclient): unlimited ready/query/list reads over http, client-capped An unlimited read never puts limit=0 on the wire, which a non-loopback bd serve refuses ("unlimited reads are loopback-only"): - bd ready --limit 0 (the WorkFilter bridge) sent no limit at all, so the server's 100-row ready default silently capped every routed-pool queue. - Reader.Ready and Querier.Query with Limit 0 sent limit=0 and got a 400 from a non-loopback server. listReadyWork and queryIssues publish no cursor, so an unlimited read is one request bounded at the client cap plus one (one server read, so one snapshot). listIssues keeps its cursor walk, now bounded by the same cap (per-page reads; the snapshot semantics are documented in unlimited_read.go). Past BEADS_HTTP_UNLIMITED_READ_CAP (default 10000) the read is refused with *UnlimitedReadCapError, never truncated; a server that reports has_more below the requested bound is refused the same way. A dispatch tripwire refuses limit=0 on the three listing operations, so a new path that forgets fails loudly instead of only off loopback. Served tests run against a server in --allow-non-loopback mode with 150 rows: ready (role, bridge, bridge+total), query and list at limit 0 return every row with a bounded limit on every request, and the cap refuses with the typed error. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): drop the three marks S15 retires With unlimited reads served (one request bounded at the cap plus one for ready and query), gc-ready-unlimited, gc-ready-pool-demand-unlimited and gc-query-wisps-unlimited return every row on the reference profile instead of the server's default page or the loopback-only refusal. mc-list-id stays pending on S15b (--id). Co-Authored-By: Claude Opus 5.5 * docs(changelog): unlimited reads over http and BEADS_HTTP_UNLIMITED_READ_CAP Co-Authored-By: Claude Opus 5.5 * fix(httpclient): uncap the list walk and bound the ready-count fallback Review fixes for #7497 (review loop iterations 1 and 2). F1 (major): the client cap no longer applies to an unlimited list. listIssues publishes a keyset cursor, so no response is larger than a page; the walk is back to base, bounded only by --max-rows / BEADS_MAX_ROWS as on every local backend. The cap stays on the cursorless reads (ready, query), whose whole answer is one response the server buffers. The CHANGELOG, unlimited_read.go and the env doc say so. F2 (major): externaldeps.CountReadyWork's no-counter fallback, which an http store reaches whenever its server does not apply the external-dependency policy (every released server), asked for the whole ready set to report one number: over http, up to the cap's full rows, and a refusal past it that text `bd ready` swallows into a missing footer. It now asks for a one-row Lite page; the bridge pairs it with the server's countReadyWork, and refetchedReadyTotal still sizes the externally blocked case. F3: TestCheckUnlimitedPage pins a full cap+1 page that also reports has_more as the cap's refusal, not a truncation, and the env test accepts the inclusive 1000000 bound. Served: the cap test adds the unlimitedRows-2 case (the server's 149 rows with has_more), the list is pinned as uncapped with MaxRows its only bound, and the bridge's MaxRows-vs-cap precedence is pinned with the limit on the wire. F5: readyBridgePage sends MaxRows+1 rather than cap+1 when MaxRows is the tighter bound (tightenUnlimitedPage). The MaxRows check still runs first, so an answer past it keeps its *ErrTooManyRows attribution. F6: the decorator and test comments describe the cap+1 bound; the refusal reads "narrow the query or raise BEADS_HTTP_UNLIMITED_READ_CAP"; the Ready and Query Limit contracts name the http exception; mc-list-id's pending reason drops the S15 --limit 0 clause and keeps the --id refusal. F7: unlimited_read.go states that the tripwire guards Store.dispatch only, so a direct wire.Client caller (readyClaimPage) must send a positive limit itself. Not taken: F4 (an unlimited walk that meets has_more with no cursor ends as a success rather than a ServerTruncated refusal). After F1 the walk is base code again, the shape needs a skewed or older peer, and the change would land in fetchIssuePages (cyclomatic 21, cognitive 34), turning an untouched base function into a changed one in the major complexity band. Left as a follow-up. Iteration 2, N1: F5 reached only the raw bridge. `bd ready` reads through the external-dependency decorator, and against a server that leaves that policy to the client, one externally blocked issue made readyExclusion take MaxRows off the unlimited inner query. The bridge then asked for cap+1 rows: past the cap it refused with the read cap's error (exit 1, "raise BEADS_HTTP_UNLIMITED_READ_CAP") rather than --max-rows' (exit 2), and under it fetched the whole ready set to refuse on the rows kept. An unlimited inner query now keeps MaxRows plus one row per blocked issue: past that many rows more than MaxRows are kept whatever the drop takes, so the bridge sizes the request by it under the cap, and innerCapError hands the refusal back as the caller's own cap. The suggested inner Limit of MaxRows+blocked+1 was not taken: a bounded request skips the cap and its server-truncation check, so with any external blocker a --max-rows above the cap would fetch past it. TestServedReadyBridgeMaxRowsSizesTheRequest pins the request and the refusal through externaldeps.New with one blocker, the cap still in force past MaxRows plus the blocked rows; TestServedReadyMaxRowsCountsTheRowsKept adds an unlimited case whose refusal names the caller's cap on all three decorator reads. N4: QueryRequest.Limit's "NOTHING BACKSTOPS IT" names the http exception in its lead, as ReadyRequest.Limit does. Not taken in iteration 2, as non-gating follow-ups: N2 (refetchedReadyTotal's Limit = total+blocked read sits outside the cap; pre-existing), N3 (the list walk's has_more-without-cursor end, the deferred F4), N5 (prefixing the malformed-cap error with its operation; it already names the variable and its value). Validation: the served tier (BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1) for internal/httpclient and the command corpus; go test for externaldeps, issueops and workapi. Each new test was mutation-checked red: the zero-limit count, the tightening removed, the MaxRows check removed, the PR's list cap restored, and a cap+1 has_more page read as a truncation. Iteration 2: the served internal/httpclient ready, unlimited-read and external-policy tests (34 PASS, 0 SKIP) and go test for externaldeps, issueops and workapi; the new pins go red with MaxRows taken off again (the pre-fix code), with no re-attribution, with no growth by the blocked rows, with the suggested inner Limit, and with Found left unadjusted. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 16 + backend/http/corpus/testdata/corpus.yaml | 5 +- internal/httpclient/BUILD.bazel | 3 + internal/httpclient/bridge.go | 14 + internal/httpclient/dispatch.go | 5 + internal/httpclient/encode/encode.go | 3 + internal/httpclient/role_querier.go | 9 + internal/httpclient/role_reader.go | 11 + .../served_external_dependency_policy_test.go | 41 +- .../httpclient/served_unlimited_read_test.go | 511 ++++++++++++++++++ internal/httpclient/unlimited_read.go | 195 +++++++ internal/httpclient/unlimited_read_test.go | 128 +++++ internal/storage/externaldeps/decorator.go | 73 ++- .../storage/externaldeps/decorator_test.go | 75 ++- issueops/querier.go | 25 +- issueops/reader.go | 4 +- 16 files changed, 1060 insertions(+), 58 deletions(-) create mode 100644 internal/httpclient/served_unlimited_read_test.go create mode 100644 internal/httpclient/unlimited_read.go create mode 100644 internal/httpclient/unlimited_read_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 986e0cb64..3eea684e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 multi-tenant embedder serving many workspaces needs, where a process-global credential cannot stand in for one tenant's own. EXPERIMENTAL, pin an exact beads version. +- Unlimited ready and query reads over the HTTP backend (`bd ready --limit + 0`, `bd query --limit 0`, and every library read asking for no limit) + return every row against any server, bounded on the client by the new + `BEADS_HTTP_UNLIMITED_READ_CAP` (a positive integer up to 1000000; default + 10000). The client never puts `limit=0` on the wire, which a `bd serve + --allow-non-loopback` refuses and which an older server answered with its + default page, silently capping the result: an unlimited ready or query + read is one request bounded at the cap plus one (one consistent snapshot). + Past the cap, or when a server reports more rows below the requested + bound, the read is refused with a typed error + (`httpclient.ErrUnlimitedReadCap`) naming the cap, never truncated. An + unlimited `bd list` is unchanged: it walks its keyset cursor a page at a + time and is bounded only by `--max-rows` / `BEADS_MAX_ROWS`. Unlike a + malformed `BEADS_MAX_ROWS`, which is ignored with a warning, a malformed + cap value fails every unlimited ready or query read rather than falling + back to the default. ### Changed diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index ded7e9e30..99a9a9c8b 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -179,7 +179,6 @@ shapes: source: gc bdstore.go bdReadyArgs (BdStore.Ready, ready --json --limit 0) args: [ready, --json, --limit, "0"] compare: ids - pending: {slice: "S15", reason: "ready --limit 0 omits limit and the server default page silently caps the answer (B3 #2)"} - id: gc-ready-ephemeral-assignee-unlimited source: gc bdstore.go bdReadyArgs with --include-ephemeral --assignee args: [ready, --json, --include-ephemeral, --assignee, worker, --limit, "0"] @@ -192,7 +191,6 @@ shapes: source: gc workquery.go bdReadyPoolDemandShell (work_query, --include-ephemeral --limit 0) args: [ready, --include-ephemeral, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit, "0"] compare: ids - pending: {slice: "S15", reason: "ready --limit 0 silently capped by the server's default page (B3 #2)"} - id: gc-ready-workflow-migration source: gc workquery.go bdReadyPoolDemandMigrationShell (--sort oldest --limit 0) args: [ready, --include-ephemeral, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit, "0"] @@ -286,7 +284,7 @@ shapes: - id: mc-list-id source: mc skills/pr-review (list --all --id --json --limit) args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] - pending: {slice: "S6c+S15+S15b", reason: "list stub (B3 #1); --limit 0 (B3 #3); --id refused by the list encoder (B3 #11)"} + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --id refused by the list encoder (B3 #11)"} - id: mc-list-assignee-status source: mc review summaries (list --assignee --json --limit --status) args: [list, --assignee, alice, --json, --limit, "10", --status, open] @@ -316,7 +314,6 @@ shapes: source: gc workquery.go bdQueryEphemeralStatusShell (--limit=0, wisp tier) args: [query, --json, "ephemeral=true AND status=open", --limit=0] compare: ids - pending: {slice: "S15", reason: "query --limit=0 sends limit=0 and a non-loopback server refuses unlimited reads (B3 #3)"} - id: mc-query source: mc pr_merge.py (bd query) args: [query, --json, "status=open AND priority<=1", --limit, "50"] diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 4b0f74f0e..ea7f30ec2 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -51,6 +51,7 @@ go_library( "sweeper.go", "target.go", "target_seam.go", + "unlimited_read.go", "unsupported.go", "unsupported_gen.go", "vocabulary.go", @@ -156,10 +157,12 @@ go_test( "served_relations_test.go", "served_releaser_test.go", "served_test.go", + "served_unlimited_read_test.go", "served_wrong_server_test.go", "store_dependent_test.go", "store_test.go", "target_test.go", + "unlimited_read_test.go", "watch_test.go", "wave2b_roles_test.go", "wave2c_claimnext_test.go", diff --git a/internal/httpclient/bridge.go b/internal/httpclient/bridge.go index f6a1baccd..32f804d9d 100644 --- a/internal/httpclient/bridge.go +++ b/internal/httpclient/bridge.go @@ -117,6 +117,17 @@ func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.Wor if err != nil { return nil, false, s.inexpressible(op, err) } + // A legacy filter spells unlimited as Limit <= 0, and the encoder therefore + // emits no `limit` for it — which the server reads as ITS default of 100, + // not as unlimited. That was a silent cap on every `bd ready --limit 0` + // (and every gc routed-pool queue read through it). The pager decides the + // bound instead: the client cap plus one, refused past the cap rather than + // truncated, and never the wire's own `limit=0` (unlimited_read.go). + readCap, err := boundUnlimitedPage(params, filter.Limit <= 0) + if err != nil { + return nil, false, err + } + readCap = tightenUnlimitedPage(params, readCap, filter.MaxRows) var body apigen.ReadyPage if err := s.dispatch(ctx, wire.Request{ Op: wire.OpListReadyWork, @@ -141,6 +152,9 @@ func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.Wor Source: filter.MaxRowsSource, } } + if err := checkUnlimitedPage(wire.OpListReadyWork, readCap, len(rows), body.HasMore); err != nil { + return nil, false, err + } return rows, body.HasMore, nil } diff --git a/internal/httpclient/dispatch.go b/internal/httpclient/dispatch.go index 8322da940..a168668d8 100644 --- a/internal/httpclient/dispatch.go +++ b/internal/httpclient/dispatch.go @@ -48,6 +48,11 @@ func (s *Store) dispatch(ctx context.Context, req wire.Request, out any) error { if s.wire == nil { return fmt.Errorf("%w: cannot dial %s", ErrNoTransport, s.target) } + // Before the preflight, so the tripwire costs no handshake: `limit=0` is + // refused off loopback, and this client never sends it (unlimited_read.go). + if err := refuseUnlimitedOnWire(req); err != nil { + return err + } if err := s.wire.Preflight(ctx, req.Op); err != nil { return err } diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 43d61f55d..41be7e31e 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -720,6 +720,9 @@ func (b *builder) intPtr(param string, value *int) { // intPositive emits a plain int only when it bounds something. The legacy // filters spell "no bound" as 0, so a zero here is an absent parameter rather // than the wire's `limit=0`, which means unlimited and is refused off loopback. +// An absent `limit` is the SERVER's default, not unlimited, so a caller that +// means unlimited must not stop here: the ready bridge's pager writes the +// client-capped bound itself (httpclient/unlimited_read.go). func (b *builder) intPositive(param string, value int) { if value > 0 { b.v.Set(param, strconv.Itoa(value)) diff --git a/internal/httpclient/role_querier.go b/internal/httpclient/role_querier.go index 0129a99f1..3d61af937 100644 --- a/internal/httpclient/role_querier.go +++ b/internal/httpclient/role_querier.go @@ -35,6 +35,12 @@ func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issu if err != nil { return issueops.IssuePage{}, q.store.inexpressible("Querier.Query", err) } + // An explicit Limit of 0 is bounded at the client cap rather than sent as + // `limit=0`, which a non-loopback server refuses (unlimited_read.go). + readCap, err := boundUnlimitedPage(params, req.Limit != nil && *req.Limit == 0) + if err != nil { + return issueops.IssuePage{}, err + } var body apigen.QueryPage if err := q.store.dispatch(ctx, wire.Request{ Op: wire.OpQueryIssues, @@ -44,6 +50,9 @@ func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issu }, &body); err != nil { return issueops.IssuePage{}, err } + if err := checkUnlimitedPage(wire.OpQueryIssues, readCap, len(body.Items), body.HasMore); err != nil { + return issueops.IssuePage{}, err + } // NEVER projected: issueops.QueryRequest has no Brief member, so this role // cannot ask for one and its rows are always whole. The `false` is the // compiler asking the question rather than a value to tune — if the query diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index 8d3333ab5..d1758a083 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -32,11 +32,19 @@ type httpReader struct{ store *Store } // server's own has_more ARE the page contract. An Offset is refused by the // encoder rather than dropped — ready work carries no keyset position, so there // is no honest way to page it (E-ReadyRequest.Offset). +// +// An explicit Limit of 0 (unlimited) is NOT sent as `limit=0`: it becomes one +// request bounded at the client cap plus one, refused past the cap rather than +// truncated (unlimited_read.go). func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { params, err := encode.ReadyParams(req) if err != nil { return issueops.IssuePage{}, r.store.inexpressible("Reader.Ready", err) } + readCap, err := boundUnlimitedPage(params, req.Limit != nil && *req.Limit == 0) + if err != nil { + return issueops.IssuePage{}, err + } var body apigen.ReadyPage if err := r.store.dispatch(ctx, wire.Request{ Op: wire.OpListReadyWork, @@ -46,6 +54,9 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue }, &body); err != nil { return issueops.IssuePage{}, err } + if err := checkUnlimitedPage(wire.OpListReadyWork, readCap, len(body.Items), body.HasMore); err != nil { + return issueops.IssuePage{}, err + } return issueops.IssuePage{Items: wireRows(body.Items, req.Brief), HasMore: body.HasMore}, nil } diff --git a/internal/httpclient/served_external_dependency_policy_test.go b/internal/httpclient/served_external_dependency_policy_test.go index e530a3085..3b11a08bd 100644 --- a/internal/httpclient/served_external_dependency_policy_test.go +++ b/internal/httpclient/served_external_dependency_policy_test.go @@ -124,12 +124,15 @@ func TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability // TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage pins, // against a real served server, the shape the decorator once misread: an -// unlimited ready listing over http comes back as the server's default page -// (the wire omits a zero limit) while its total counts the whole ready set. -// Text-mode `bd ready` counts through CountReadyWork, which always fetches -// unlimited, so an externally blocked issue sorted past that page stayed in -// its count when the decorator took the zero limit for a window holding every -// row. +// unlimited ready listing over http used to come back as the server's default +// page (the wire omitted a zero limit) while its total counted the whole ready +// set. Text-mode `bd ready` counted through CountReadyWork's unlimited fetch, +// so an externally blocked issue sorted past that page stayed in its count when +// the decorator took the zero limit for a window holding every row. An +// unlimited ready read now returns every row (one request bounded at the client +// cap, unlimited_read.go), and CountReadyWork asks for one row and lets the +// server count the rest; the count and the default page must still leave that +// issue out. func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *testing.T) { env := newServedEnv(t, "hixp") ctx := t.Context() @@ -165,19 +168,20 @@ func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *t if err != nil { t.Fatalf("BuildReadyCountFilter: %v", err) } - // The premise this test pins: undecorated, the unlimited listing is the - // default page, the blocked issue is not on it, and the total counts it. + // The premise this test pins: undecorated, the unlimited listing holds + // every ready row (no longer only the server's default page), and the + // blocked issue sorts past where the default page ends. rows, total, err := env.subject.GetReadyWorkWithCountsAndTotal(ctx, countFilter) if err != nil { t.Fatalf("undecorated GetReadyWorkWithCountsAndTotal over http: %v", err) } - if len(rows) != workapi.DefaultReadyLimit || total != workapi.DefaultReadyLimit+2 { - t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want the %d-row default page, total %d; this test's premise no longer holds", - len(rows), total, workapi.DefaultReadyLimit, workapi.DefaultReadyLimit+2) + if len(rows) != workapi.DefaultReadyLimit+2 || total != workapi.DefaultReadyLimit+2 { + t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want every ready row, %d and total %d; this test's premise no longer holds", + len(rows), total, workapi.DefaultReadyLimit+2, workapi.DefaultReadyLimit+2) } - for _, row := range rows { - if row != nil && row.Issue != nil && row.ID == blockedID { - t.Fatalf("%q is on the default page; this test needs it past the page", blockedID) + for i, row := range rows { + if row != nil && row.Issue != nil && row.ID == blockedID && i < workapi.DefaultReadyLimit { + t.Fatalf("%q is row %d, inside the default page; this test needs it past the page", blockedID, i) } } @@ -443,7 +447,8 @@ func TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter(t *test // fetches a window wider than the limit to make room for the rows it drops, // and the http bridge caps the rows it fetched, so the cap counted rows the // caller was never handed. A SQL store caps the page it delivers, and so must -// this path. +// this path. An unlimited read keeps the cap on its request, grown by the rows +// the decorator may drop, and refuses in the caller's name past it. func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { env := newServedEnv(t, "hixm") ctx := t.Context() @@ -478,8 +483,12 @@ func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { {name: "cap equals the limit", limit: 2, maxRows: 2, want: free[:2]}, // The window holds all four rows, three of them kept. {name: "cap below the limit", limit: 5, maxRows: 3, want: free}, - // The server's default page, again all four. + // The request is sized by the cap grown by the blocked row: all four + // rows, three of them kept. {name: "unlimited", limit: 0, maxRows: 3, want: free}, + // The four rows outnumber the cap grown by the blocked row, so at + // least three would be kept, and the caller's cap of two fires. + {name: "unlimited, kept rows exceed the cap", limit: 0, maxRows: 2}, // The three rows kept exceed the caller's cap, which still fires. {name: "kept rows exceed the cap", limit: 5, maxRows: 2}, } { diff --git a/internal/httpclient/served_unlimited_read_test.go b/internal/httpclient/served_unlimited_read_test.go new file mode 100644 index 000000000..0e636251c --- /dev/null +++ b/internal/httpclient/served_unlimited_read_test.go @@ -0,0 +1,511 @@ +//go:build cgo + +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/url" + "slices" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage/externaldeps" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// Unlimited reads against a server in NON-LOOPBACK mode (S15). +// +// The in-process server binds 127.0.0.1 — tests never listen beyond loopback — +// but is started with AllowNonLoopback, which is exactly the switch the +// server's unlimited-read refusal reads (httpapi allowUnlimited). So every +// `limit=0` this client sent would meet the production 400, and every answer +// below that comes back whole is proof that none was sent. +// +// The fixture is unlimitedRows rows: more than the server's ready default +// (workapi.DefaultReadyLimit, 100), so a read that silently fell back to that +// default — `bd ready --limit 0` before S15 — comes back short and fails the +// count, and more than one walkPageSize, so the list walk crosses pages. + +const unlimitedRows = 150 + +// nonLoopbackMode serves the whole surface as a --allow-non-loopback server +// would. InsecureNoAuth is the posture rule's required waiver for a +// non-loopback server with no token file; the listener itself stays on +// 127.0.0.1. +func nonLoopbackMode(cfg *httpapi.Config) { + cfg.AllowNonLoopback = true + cfg.InsecureNoAuth = true +} + +// limitSpy records the `limit` of every issue-listing request that reaches the +// wire, after dispatch's own tripwire. +type limitSpy struct { + mu sync.Mutex + sent []string // "op limit=" +} + +func (s *limitSpy) record(req *wire.Request) { + if !unlimitedSpellingOps[req.Op] { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.sent = append(s.sent, fmt.Sprintf("%s limit=%s", req.Op, req.Query.Get("limit"))) +} + +func (s *limitSpy) reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.sent = nil +} + +func (s *limitSpy) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.sent...) +} + +// requestsFor is requests narrowed to one operation's. +func (s *limitSpy) requestsFor(op string) []string { + var got []string + for _, r := range s.requests() { + if strings.HasPrefix(r, op+" ") { + got = append(got, r) + } + } + return got +} + +// assertBoundedOnWire fails if any recorded request carried limit=0 or +// no limit at all (the server's default, which is the silent-truncation leg). +func (s *limitSpy) assertBoundedOnWire(t *testing.T) { + t.Helper() + got := s.requests() + if len(got) == 0 { + t.Fatal("no listing request was recorded; the spy is not on the path") + } + for _, r := range got { + if strings.HasSuffix(r, " limit=0") || strings.HasSuffix(r, " limit=") { + t.Errorf("request %q went out unbounded (all requests: %v)", r, got) + } + } +} + +type unlimitedEnv struct { + *servedEnv + spy *limitSpy + store *Store + scope string +} + +func newUnlimitedEnv(t *testing.T) *unlimitedEnv { + t.Helper() + e := newServedEnv(t, "unl", nonLoopbackMode) + ctx := t.Context() + scope := "unlimited-scope" + issues := make([]*types.Issue, 0, unlimitedRows) + for i := range unlimitedRows { + id := fmt.Sprintf("unl-%03d", i) + issues = append(issues, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: i % 4, + IssueType: types.TypeTask, Labels: []string{scope}, + }) + } + if err := e.reference.CreateIssues(ctx, issues, "seed"); err != nil { + t.Fatalf("seed %d rows: %v", unlimitedRows, err) + } + spy := &limitSpy{} + store := New(e.subject.target, tamperedWire{WireClient: e.subject.wire, tamper: spy.record}, nil) + return &unlimitedEnv{servedEnv: e, spy: spy, store: store, scope: scope} +} + +func (u *unlimitedEnv) reader(t *testing.T) issueops.Reader { + t.Helper() + r, err := u.store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return r +} + +func (u *unlimitedEnv) querier(t *testing.T) issueops.Querier { + t.Helper() + q, err := u.store.Querier() + if err != nil { + t.Fatalf("Querier(): %v", err) + } + return q +} + +// listAll is an unlimited list: a cursor walk that only maxRows bounds (0 for +// none). It returns the row count. +func (u *unlimitedEnv) listAll(t *testing.T, maxRows int) (int, error) { + t.Helper() + page, err := u.reader(t).List(t.Context(), issueops.ListRequest{ + Labels: []string{u.scope}, Limit: ptrTo(0), MaxRows: maxRows, + }) + if err == nil && page.HasMore { + t.Error("an unlimited list reported HasMore") + } + return len(page.Items), err +} + +// readFn is one of the unlimited reads under test, returning its row count. +type readFn func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) + +// cursorlessReads are the unlimited reads that are ONE request — ready and +// query publish no cursor — and so the ones the client cap bounds. +func cursorlessReads() []struct { + name string + read readFn +} { + zero := func() *int { z := 0; return &z } + return []struct { + name string + read readFn + }{ + {"ready via the Reader role", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + page, err := u.reader(t).Ready(ctx, issueops.ReadyRequest{Labels: []string{u.scope}, Limit: zero()}) + return len(page.Items), err + }}, + {"bd ready via the WorkFilter bridge", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: zero(), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, err := u.store.GetReadyWork(ctx, filter) + return len(rows), err + }}, + {"bd ready --json via the WorkFilter bridge with a total", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: zero(), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, total, err := u.store.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err == nil && total != len(rows) { + t.Errorf("total = %d for %d rows; an unlimited page is the whole set", total, len(rows)) + } + return len(rows), err + }}, + {"query", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + page, err := u.querier(t).Query(ctx, issueops.QueryRequest{Expression: "label=" + u.scope, Limit: zero()}) + return len(page.Items), err + }}, + } +} + +// TestServedUnlimitedReadsOffLoopback: ready, query and list at limit 0 return +// every row from a non-loopback-mode server, and never put `limit=0` (or a +// missing limit) on the wire. +// +// GOES RED when the bridge sends no limit for Limit 0 (the server's 100-row +// default: 100 != 150), when the role or the querier forwards `limit=0` (the +// server's 400, or the dispatch tripwire), or when the list walk stops short of +// the fixture. +func TestServedUnlimitedReadsOffLoopback(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + + restore := walkPageSize + walkPageSize = 40 + t.Cleanup(func() { walkPageSize = restore }) + + // PREMISE 1: the server really is in non-loopback mode — a raw `limit=0` + // that bypasses the client's dispatch is refused. + var raw apigen.ReadyPage + err := u.subject.wire.Do(ctx, wire.Request{ + Op: wire.OpListReadyWork, Method: "GET", Path: wire.PathReady, + Query: url.Values{"limit": {"0"}}, + }, &raw) + if err == nil || !strings.Contains(err.Error(), "loopback-only") { + t.Fatalf("a raw limit=0 was not refused by the server (err %v); the fixture is not in non-loopback mode", err) + } + + // PREMISE 2: the fixture exceeds the server's ready default, so a read + // that fell back to it cannot pass the count below. + page, err := u.reader(t).Ready(ctx, issueops.ReadyRequest{Labels: []string{u.scope}}) + if err != nil { + t.Fatalf("default-limit ready: %v", err) + } + if len(page.Items) != workapi.DefaultReadyLimit || !page.HasMore { + t.Fatalf("default-limit ready returned %d rows (has_more %v), want the server default %d with more", + len(page.Items), page.HasMore, workapi.DefaultReadyLimit) + } + + for _, tc := range cursorlessReads() { + t.Run(tc.name, func(t *testing.T) { + u.spy.reset() + n, err := tc.read(t, ctx, u) + if err != nil { + t.Fatalf("unlimited read: %v", err) + } + if n != unlimitedRows { + t.Errorf("unlimited read returned %d rows, want all %d", n, unlimitedRows) + } + u.spy.assertBoundedOnWire(t) + }) + } + + // The list walk returns every row, and crosses pages rather than asking + // for everything at once. + t.Run("list", func(t *testing.T) { + u.spy.reset() + n, err := u.listAll(t, 0) + if err != nil { + t.Fatalf("unlimited list: %v", err) + } + if n != unlimitedRows { + t.Errorf("unlimited list returned %d rows, want all %d", n, unlimitedRows) + } + u.spy.assertBoundedOnWire(t) + if got := len(u.spy.requests()); got < unlimitedRows/walkPageSize { + t.Errorf("the unlimited list took %d requests at %d rows a page; it did not walk", got, walkPageSize) + } + }) +} + +// TestServedUnlimitedReadCapRefuses: past BEADS_HTTP_UNLIMITED_READ_CAP every +// unlimited ready or query read is refused with *UnlimitedReadCapError, never +// truncated; at the cap exactly, the whole answer comes back. An unlimited list +// is a cursor walk, and the cap does not apply to it: only MaxRows bounds it. +// +// GOES RED when a cursorless path truncates to the cap instead of refusing, or +// ignores it; when a full cap+1 page that reports has_more is taken for a +// server truncation; or when the list walk is capped. +func TestServedUnlimitedReadCapRefuses(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + + for _, tc := range cursorlessReads() { + t.Run(tc.name, func(t *testing.T) { + // At unlimitedRows-1 the cap+1 page is the whole set; at + // unlimitedRows-2 it is a full page with has_more set, which is how + // a real workspace overflows the cap. Both are the cap's refusal. + for _, readCap := range []int{unlimitedRows - 1, unlimitedRows - 2} { + t.Setenv(UnlimitedReadCapEnv, fmt.Sprint(readCap)) + u.spy.reset() + n, err := tc.read(t, ctx, u) + var capErr *UnlimitedReadCapError + if !errors.As(err, &capErr) { + t.Fatalf("cap %d: got %d rows, err %v; want *UnlimitedReadCapError", readCap, n, err) + } + if !errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("cap %d: the refusal does not match ErrUnlimitedReadCap: %v", readCap, err) + } + if capErr.Cap != readCap || capErr.Found != readCap+1 || capErr.ServerTruncated { + t.Errorf("cap %d: refusal = %+v, want Cap %d Found %d and no server truncation", + readCap, capErr, readCap, readCap+1) + } + if !strings.Contains(err.Error(), UnlimitedReadCapEnv) { + t.Errorf("cap %d: the refusal does not name %s: %v", readCap, UnlimitedReadCapEnv, err) + } + u.spy.assertBoundedOnWire(t) + } + + t.Setenv(UnlimitedReadCapEnv, fmt.Sprint(unlimitedRows)) + n, err := tc.read(t, ctx, u) + if err != nil || n != unlimitedRows { + t.Fatalf("at the cap exactly: got %d rows, err %v; want all %d", n, err, unlimitedRows) + } + + t.Setenv(UnlimitedReadCapEnv, "lots") + if _, err := tc.read(t, ctx, u); err == nil || !strings.Contains(err.Error(), UnlimitedReadCapEnv) { + t.Errorf("a malformed cap was not refused: %v", err) + } + }) + } + + t.Run("list is not capped", func(t *testing.T) { + restore := walkPageSize + walkPageSize = 40 + t.Cleanup(func() { walkPageSize = restore }) + + for _, readCap := range []string{"10", "lots"} { + t.Setenv(UnlimitedReadCapEnv, readCap) + if n, err := u.listAll(t, 0); err != nil || n != unlimitedRows { + t.Errorf("cap %q: the unlimited list returned %d rows, err %v; want all %d", readCap, n, err, unlimitedRows) + } + } + + // MaxRows is the walk's only bound, and past it the refusal is the + // caller's own. + t.Setenv(UnlimitedReadCapEnv, "10") + _, err := u.listAll(t, 100) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) || tooMany.Cap != 100 || tooMany.Found != 101 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("the unlimited list past MaxRows 100: err %v; want *ErrTooManyRows Cap 100 Found 101", err) + } + if n, err := u.listAll(t, unlimitedRows); err != nil || n != unlimitedRows { + t.Errorf("MaxRows at the fixture: %d rows, err %v; want all %d", n, err, unlimitedRows) + } + }) +} + +// TestServedReadyBridgeMaxRowsSizesTheRequest: when an unlimited bridge read +// carries a MaxRows as well, the tighter of it and the cap sizes the request, +// so a small MaxRows never makes the server buffer up to the cap, and an answer +// past it is refused in the name of the bound that fired. `bd ready` reads +// through the external-dependency decorator, which drops an externally +// blocked row itself against this server; the same holds through it, with the +// request grown by the rows it may drop and the cap still in force. +// +// GOES RED when the bridge sends the cap's limit under a tighter MaxRows, or +// reports an answer past MaxRows as the read cap's refusal; or when the +// decorator takes MaxRows off an unlimited read, does not grow it by the +// blocked rows, reports the refusal against the grown bound, or lets the +// growth lift the request past the cap. +func TestServedReadyBridgeMaxRowsSizesTheRequest(t *testing.T) { + u := newUnlimitedEnv(t) + read := func(t *testing.T, ready func(context.Context, types.WorkFilter) ([]*types.Issue, error), maxRows int) (int, error) { + t.Helper() + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: ptrTo(0), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + filter.MaxRows = maxRows + u.spy.reset() + rows, err := ready(t.Context(), filter) + return len(rows), err + } + sent := func(limit int) []string { return []string{fmt.Sprintf("%s limit=%d", wire.OpListReadyWork, limit)} } + + t.Setenv(UnlimitedReadCapEnv, "10") + _, err := read(t, u.store.GetReadyWork, 5) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) || tooMany.Cap != 5 || tooMany.Found != 6 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("MaxRows 5 under cap 10: err %v; want *ErrTooManyRows Cap 5 Found 6", err) + } + if got := u.spy.requests(); !slices.Equal(got, sent(6)) { + t.Errorf("MaxRows 5 under cap 10 sent %v, want %v", got, sent(6)) + } + + _, err = read(t, u.store.GetReadyWork, 50) + var capErr *UnlimitedReadCapError + if !errors.As(err, &capErr) || capErr.Cap != 10 || capErr.Found != 11 || capErr.ServerTruncated { + t.Errorf("MaxRows 50 over cap 10: err %v; want *UnlimitedReadCapError Cap 10 Found 11", err) + } + if got := u.spy.requests(); !slices.Equal(got, sent(11)) { + t.Errorf("MaxRows 50 over cap 10 sent %v, want %v", got, sent(11)) + } + + // Both bounds clear the fixture: every row, in the request MaxRows sized. + t.Setenv(UnlimitedReadCapEnv, "1000") + if n, err := read(t, u.store.GetReadyWork, unlimitedRows+10); err != nil || n != unlimitedRows { + t.Errorf("MaxRows %d under cap 1000: %d rows, err %v; want all %d", unlimitedRows+10, n, err, unlimitedRows) + } + if got := u.spy.requests(); !slices.Equal(got, sent(unlimitedRows+11)) { + t.Errorf("MaxRows %d under cap 1000 sent %v, want %v", unlimitedRows+10, got, sent(unlimitedRows+11)) + } + + // Through the decorator, with unl-000 externally blocked: the request asks + // for one row more per blocked issue, and past that bound more than MaxRows + // rows would still be kept, so the refusal is still the caller's own. The + // decorator's blocking state is loaded with a list walk, which is not the + // read's. + if enforced, err := u.store.ServerEnforcesExternalDependencyPolicy(t.Context()); err != nil || enforced { + t.Fatalf("ServerEnforcesExternalDependencyPolicy = %v, %v; this test needs a server that leaves the policy to the client", enforced, err) + } + if err := u.addDependency(t.Context(), &types.Dependency{ + IssueID: "unl-000", DependsOnID: "external:otherproj:cap", Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(u.store, nil, nil).GetReadyWork + + t.Setenv(UnlimitedReadCapEnv, "10") + _, err = read(t, decorated, 5) + if !errors.As(err, &tooMany) || tooMany.Cap != 5 || tooMany.Found != 6 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("decorated, MaxRows 5 under cap 10: err %v; want *ErrTooManyRows Cap 5 Found 6", err) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(7)) { + t.Errorf("decorated, MaxRows 5 under cap 10 sent %v, want %v", got, sent(7)) + } + + // MaxRows and the blocked row together reach past the cap, which sizes + // the request and refuses, as it does for the bridge alone. + _, err = read(t, decorated, 50) + if !errors.As(err, &capErr) || capErr.Cap != 10 || capErr.Found != 11 || capErr.ServerTruncated { + t.Errorf("decorated, MaxRows 50 over cap 10: err %v; want *UnlimitedReadCapError Cap 10 Found 11", err) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(11)) { + t.Errorf("decorated, MaxRows 50 over cap 10 sent %v, want %v", got, sent(11)) + } + + t.Setenv(UnlimitedReadCapEnv, "1000") + if n, err := read(t, decorated, unlimitedRows+10); err != nil || n != unlimitedRows-1 { + t.Errorf("decorated, MaxRows %d under cap 1000: %d rows, err %v; want all %d but unl-000", unlimitedRows+10, n, err, unlimitedRows-1) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(unlimitedRows+12)) { + t.Errorf("decorated, MaxRows %d under cap 1000 sent %v, want %v", unlimitedRows+10, got, sent(unlimitedRows+12)) + } +} + +// TestServedReadyCountFallbackStaysBounded: text-mode `bd ready` counts through +// the external-dependency decorator, and against a server that does not apply +// that policy itself (every released server, and this harness) the decorator +// counts over a store with no indexed counter. It asks for one row and lets +// the server count the rest, so a ready set larger than +// BEADS_HTTP_UNLIMITED_READ_CAP is counted rather than refused, and never +// downloaded to report one number. +// +// GOES RED when the count's fallback asks for the whole ready set again: past +// the cap that read is refused, and the spy sees more than the one row. +func TestServedReadyCountFallbackStaysBounded(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + t.Setenv(UnlimitedReadCapEnv, "10") + + if enforced, err := u.store.ServerEnforcesExternalDependencyPolicy(ctx); err != nil || enforced { + t.Fatalf("ServerEnforcesExternalDependencyPolicy = %v, %v; this test needs a server that leaves the policy to the client", enforced, err) + } + counter, err := externaldeps.New(u.store, nil, nil).ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + // count returns the total and the count's ready listings; the decorator's + // blocking state is loaded with a list walk, which is not the count's. + count := func(t *testing.T) (int64, []string) { + t.Helper() + u.spy.reset() + got, err := counter.CountReady(ctx, issueops.ReadyRequest{Labels: []string{u.scope}}) + if err != nil { + t.Fatalf("CountReady over %d ready rows at cap 10: %v", unlimitedRows, err) + } + return got.Total, u.spy.requestsFor(wire.OpListReadyWork) + } + + total, ready := count(t) + if total != unlimitedRows { + t.Errorf("count = %d, want %d", total, unlimitedRows) + } + if want := []string{wire.OpListReadyWork + " limit=1"}; !slices.Equal(ready, want) { + t.Errorf("the count listed %v, want %v: one row, the server counting the rest", ready, want) + } + + // An externally blocked issue makes the decorator size the filtered set + // itself, at an explicit limit the cap does not apply to. + if err := u.addDependency(ctx, &types.Dependency{ + IssueID: "unl-000", DependsOnID: "external:otherproj:cap", Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + if total, _ := count(t); total != unlimitedRows-1 { + t.Errorf("count with unl-000 externally blocked = %d, want %d", total, unlimitedRows-1) + } + u.spy.assertBoundedOnWire(t) +} diff --git a/internal/httpclient/unlimited_read.go b/internal/httpclient/unlimited_read.go new file mode 100644 index 000000000..b607fd0a1 --- /dev/null +++ b/internal/httpclient/unlimited_read.go @@ -0,0 +1,195 @@ +package httpclient + +import ( + "errors" + "fmt" + "net/url" + "os" + "strconv" + "strings" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// UNLIMITED READS OVER HTTP. +// +// Every issue listing on the v0 wire spells "unlimited" as `limit=0`, and a +// server started with --allow-non-loopback refuses that spelling outright +// (400, "unlimited reads are loopback-only; pass an explicit limit"). The +// document's recovery is client-side: ask for an explicit limit, and page. So +// this client NEVER puts `limit=0` on the wire, whatever the server's bind +// mode — one spelling for every server is what keeps a loopback-tested caller +// from meeting the refusal for the first time in production. Store.dispatch +// holds the line (refuseUnlimitedOnWire), so a new read path that forgets is a +// loud client error rather than a silent difference between a laptop and a +// fleet. The tripwire guards Store.dispatch only: a caller of the wire.Client +// itself is outside it and must send a positive limit on its own, as the +// down-level claimer's listReadyWork page does (readyClaimPage). +// +// What replaces `limit=0` depends on what paging the operation publishes: +// +// - listIssues publishes a keyset cursor, so an unlimited `bd list` is the +// cursor walk in list_walk.go, at walkPageSize rows a request. No response +// is larger than a page, so the walk is NOT capped here: as on every local +// backend, its only bound is the caller's --max-rows / BEADS_MAX_ROWS. +// - listReadyWork and queryIssues publish NO cursor and no offset (their +// pages are has_more-only by design: a ready set and a predicate query are +// assembled outside the database's keyset order). There is nothing to walk, +// so an unlimited read is ONE request bounded at the cap plus one: the +// overage row is what proves the cap fired. +// +// THE CAP. Every unlimited ready or query read is bounded on the client at +// UnlimitedReadCapEnv rows (DefaultUnlimitedReadCap when unset), because its +// whole answer is one response the server buffers. Past it the read is +// REFUSED with *UnlimitedReadCapError — never truncated. A truncated +// "unlimited" answer is the defect this file exists to remove (the server's +// 100-row ready default used to cap gc's routed-pool queues silently), so the +// only two answers are "every row" and "a typed refusal naming the cap". +// +// SNAPSHOT SEMANTICS, stated per shape because they differ: +// +// - ready and query: one request is one server read, so the answer is one +// consistent snapshot of the ready set / the matching set. +// - list: the cursor walk is one server read PER PAGE, not a snapshot. The +// keyset is strict, so no row is returned twice, and a row that exists and +// matches the filter for the whole walk is returned exactly once. A row +// created during the walk, or one whose status/fields change so that it +// starts or stops matching, may or may not appear — the answer is "the rows +// that matched at the moment each page was read". That is the same promise +// a paging caller of the wire gets, and it is the reason the ready shape +// (whose consumers act on the whole set at once) is NOT emulated through a +// list walk. + +// UnlimitedReadCapEnv bounds how many rows one unlimited ready or query read +// over HTTP may return before it is refused (an unlimited list is a paged +// walk and is not capped). A positive integer; unset means +// DefaultUnlimitedReadCap. A malformed value is an error on every unlimited +// read rather than a silent fall back to the default, so a typo cannot quietly +// re-impose a bound the operator meant to raise. +const UnlimitedReadCapEnv = "BEADS_HTTP_UNLIMITED_READ_CAP" + +// DefaultUnlimitedReadCap is the cap when UnlimitedReadCapEnv is unset. +const DefaultUnlimitedReadCap = 10000 + +// maxUnlimitedReadCap is the largest cap UnlimitedReadCapEnv accepts. It is a +// sanity bound on the operator's knob, not a tuning value: the request asks +// the server to buffer cap+1 rows in one response. +const maxUnlimitedReadCap = 1_000_000 + +// ErrUnlimitedReadCap is the sentinel every *UnlimitedReadCapError matches +// under errors.Is. +var ErrUnlimitedReadCap = errors.New("unlimited read exceeds the http client cap") + +// UnlimitedReadCapError refuses an unlimited ready or query read whose answer +// is larger than the client cap, or whose server would not hand back the +// whole answer. +type UnlimitedReadCapError struct { + // Op is the wire operation (listReadyWork or queryIssues). + Op string + // Cap is the bound in force. + Cap int + // Found is how many rows had arrived when the refusal fired: Cap+1 when the + // cap fired, or the page length when the server truncated (see below). + Found int + // ServerTruncated marks the other refusal: the server returned no more than + // the cap yet reported has_more, so it bounded the page below what was + // asked. Returning that page would be the silent truncation this refusal + // exists to prevent. + ServerTruncated bool +} + +func (e *UnlimitedReadCapError) Error() string { + if e.ServerTruncated { + return fmt.Sprintf("unlimited %s over http: the server returned %d rows with more remaining, "+ + "below the requested bound of %d; refusing a truncated answer (narrow the query)", + e.Op, e.Found, e.Cap+1) + } + return fmt.Sprintf("unlimited %s over http: more than %d rows match (%s=%d); "+ + "narrow the query or raise %s", + e.Op, e.Cap, UnlimitedReadCapEnv, e.Cap, UnlimitedReadCapEnv) +} + +func (e *UnlimitedReadCapError) Unwrap() error { return ErrUnlimitedReadCap } + +// unlimitedReadCap reads UnlimitedReadCapEnv. +func unlimitedReadCap() (int, error) { + raw := strings.TrimSpace(os.Getenv(UnlimitedReadCapEnv)) + if raw == "" { + return DefaultUnlimitedReadCap, nil + } + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > maxUnlimitedReadCap { + return 0, fmt.Errorf("%s=%q: want an integer between 1 and %d", UnlimitedReadCapEnv, raw, maxUnlimitedReadCap) + } + return n, nil +} + +// boundUnlimitedPage writes the bounded `limit` that replaces an unlimited +// read on a cursorless listing, returning the cap in force, or 0 when the +// request is not unlimited and goes out unchanged. +// +// params is the encoder's output and is modified in place. The encoder maps +// fields to the wire's own spelling (`limit=0` for an explicit zero, nothing +// for a legacy zero); how an unlimited read is honored is the pager's +// decision, made here. +func boundUnlimitedPage(params url.Values, unlimited bool) (int, error) { + if !unlimited { + return 0, nil + } + readCap, err := unlimitedReadCap() + if err != nil { + return 0, err + } + params.Set("limit", strconv.Itoa(readCap+1)) + return readCap, nil +} + +// tightenUnlimitedPage lets a caller's MaxRows size an unlimited read's +// request when it is the tighter bound, returning the bound now in force. +// Past MaxRows the answer is refused either way, so sending cap+1 would make +// the server buffer up to the cap's rows only for the caller to throw them +// away. The caller's own MaxRows check must run before checkUnlimitedPage, so +// an answer over it keeps its *ErrTooManyRows attribution. +func tightenUnlimitedPage(params url.Values, readCap, maxRows int) int { + if readCap == 0 || maxRows <= 0 || maxRows >= readCap { + return readCap + } + params.Set("limit", strconv.Itoa(maxRows+1)) + return maxRows +} + +// checkUnlimitedPage applies the cap to the one page a cursorless unlimited +// read received. readCap 0 means the read was bounded by the caller and the +// page is returned as the server shaped it. +func checkUnlimitedPage(op string, readCap, rows int, hasMore bool) error { + switch { + case readCap == 0: + return nil + case rows > readCap: + return &UnlimitedReadCapError{Op: op, Cap: readCap, Found: rows} + case hasMore: + return &UnlimitedReadCapError{Op: op, Cap: readCap, Found: rows, ServerTruncated: true} + } + return nil +} + +// errUnlimitedOnWire is the dispatch tripwire's answer: a client bug, never a +// server condition, so it names no server and is not retried. +var errUnlimitedOnWire = errors.New("httpclient: refusing to send limit=0 (unlimited) on the wire; an unlimited read must be paged or bounded by the client cap") + +// unlimitedSpellingOps are the operations whose `limit=0` means unlimited and +// is refused off loopback. +var unlimitedSpellingOps = map[string]bool{ + wire.OpListReadyWork: true, + wire.OpListIssues: true, + wire.OpQueryIssues: true, +} + +// refuseUnlimitedOnWire is dispatch's guard that `limit=0` never leaves this +// client on an operation that reads it as unlimited. +func refuseUnlimitedOnWire(req wire.Request) error { + if unlimitedSpellingOps[req.Op] && req.Query != nil && req.Query.Get("limit") == "0" { + return fmt.Errorf("%w (%s)", errUnlimitedOnWire, req.Op) + } + return nil +} diff --git a/internal/httpclient/unlimited_read_test.go b/internal/httpclient/unlimited_read_test.go new file mode 100644 index 000000000..b5a842094 --- /dev/null +++ b/internal/httpclient/unlimited_read_test.go @@ -0,0 +1,128 @@ +package httpclient + +import ( + "context" + "errors" + "net/http" + "net/url" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// TestDispatchRefusesUnlimitedOnTheWire pins the tripwire: `limit=0` on an +// operation that reads it as unlimited never leaves dispatch, and the +// refusal is made before any transport is consulted. +// +// GOES RED when refuseUnlimitedOnWire is removed from dispatch. +func TestDispatchRefusesUnlimitedOnTheWire(t *testing.T) { + s := New(Target{BaseURL: &url.URL{Scheme: "http", Host: "127.0.0.1:1"}}, failingWire{t: t}, nil) + for _, op := range []string{wire.OpListReadyWork, wire.OpListIssues, wire.OpQueryIssues} { + err := s.dispatch(t.Context(), wire.Request{ + Op: op, Method: http.MethodGet, Path: "/x", Query: url.Values{"limit": {"0"}}, + }, nil) + if !errors.Is(err, errUnlimitedOnWire) { + t.Errorf("%s with limit=0: err = %v, want errUnlimitedOnWire", op, err) + } + } + // A bounded limit, and an operation whose limit means something else, pass. + for _, req := range []wire.Request{ + {Op: wire.OpListIssues, Query: url.Values{"limit": {"10"}}}, + {Op: wire.OpCountReadyWork, Query: url.Values{"limit": {"0"}}}, + } { + if err := refuseUnlimitedOnWire(req); err != nil { + t.Errorf("%s %v: unexpected refusal %v", req.Op, req.Query, err) + } + } +} + +// failingWire fails the test if dispatch ever reaches it. +type failingWire struct { + WireClient + t *testing.T +} + +func (w failingWire) Preflight(_ context.Context, op string) error { + w.t.Errorf("dispatch reached Preflight for %s; the tripwire must fire first", op) + return errors.New("unreachable") +} + +// TestCheckUnlimitedPage covers the two refusals of a cursorless unlimited +// read, including the one no served fixture can produce: a server that +// truncates below the requested bound and says so with has_more. A full +// cap+1 page that also reports has_more is how a real workspace overflows +// the cap, and it is the cap's refusal, not a truncation. +func TestCheckUnlimitedPage(t *testing.T) { + for _, tc := range []struct { + name string + readCap int + rows int + hasMore bool + truncated bool + refused bool + }{ + {"bounded read passes as shaped", 0, 5, true, false, false}, + {"within the cap", 10, 10, false, false, false}, + {"over the cap", 10, 11, false, false, true}, + {"over the cap with more remaining", 10, 11, true, false, true}, + {"server truncated below the bound", 10, 4, true, true, true}, + } { + err := checkUnlimitedPage(wire.OpQueryIssues, tc.readCap, tc.rows, tc.hasMore) + var capErr *UnlimitedReadCapError + if got := errors.As(err, &capErr); got != tc.refused { + t.Fatalf("%s: refused = %v (%v), want %v", tc.name, got, err, tc.refused) + } + if tc.refused && capErr.ServerTruncated != tc.truncated { + t.Errorf("%s: ServerTruncated = %v, want %v", tc.name, capErr.ServerTruncated, tc.truncated) + } + } +} + +// TestTightenUnlimitedPage pins when a caller's MaxRows sizes the request of +// an unlimited read: only when it is tighter than the cap. A bounded read +// (readCap 0) goes out exactly as the caller shaped it. +func TestTightenUnlimitedPage(t *testing.T) { + for _, tc := range []struct { + name string + readCap int + maxRows int + sent string // the limit before the call + wantBound int + wantLimit string + }{ + {"bounded read", 0, 5, "7", 0, "7"}, + {"no MaxRows", 10, 0, "11", 10, "11"}, + {"MaxRows above the cap", 10, 50, "11", 10, "11"}, + {"MaxRows at the cap", 10, 10, "11", 10, "11"}, + {"MaxRows below the cap", 10, 5, "11", 5, "6"}, + } { + params := url.Values{"limit": {tc.sent}} + if got := tightenUnlimitedPage(params, tc.readCap, tc.maxRows); got != tc.wantBound { + t.Errorf("%s: bound = %d, want %d", tc.name, got, tc.wantBound) + } + if got := params.Get("limit"); got != tc.wantLimit { + t.Errorf("%s: limit = %q, want %q", tc.name, got, tc.wantLimit) + } + } +} + +func TestUnlimitedReadCapEnv(t *testing.T) { + t.Setenv(UnlimitedReadCapEnv, "") + if n, err := unlimitedReadCap(); err != nil || n != DefaultUnlimitedReadCap { + t.Errorf("unset: %d, %v; want the default %d", n, err, DefaultUnlimitedReadCap) + } + t.Setenv(UnlimitedReadCapEnv, " 250 ") + if n, err := unlimitedReadCap(); err != nil || n != 250 { + t.Errorf("250: %d, %v", n, err) + } + t.Setenv(UnlimitedReadCapEnv, "1000000") + if n, err := unlimitedReadCap(); err != nil || n != maxUnlimitedReadCap { + t.Errorf("the inclusive maximum: %d, %v; want %d", n, err, maxUnlimitedReadCap) + } + for _, bad := range []string{"0", "-1", "ten", "1000001"} { + t.Setenv(UnlimitedReadCapEnv, bad) + if _, err := unlimitedReadCap(); err == nil { + t.Errorf("%q was accepted", bad) + } + } +} diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 724bbfda4..5c24bb39f 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -2,6 +2,7 @@ package externaldeps import ( "context" + "errors" "fmt" "slices" "sort" @@ -226,7 +227,7 @@ func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*t queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) issues, err := s.inner.GetReadyWork(ctx, queryFilter) if err != nil { - return nil, err + return nil, innerCapError(err, filter, blockedIDs) } kept := dropBlockedIssues(issues, blockedIDs, filter.Limit) if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { @@ -251,7 +252,7 @@ func (s *Store) readyWorkWithCounts(ctx context.Context, filter types.WorkFilter queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) rows, err := s.inner.GetReadyWorkWithCounts(ctx, queryFilter) if err != nil { - return nil, err + return nil, innerCapError(err, filter, blockedIDs) } kept, _ := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { @@ -291,13 +292,21 @@ func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types // decorator has to drop. Whichever path is taken, the policy itself is never // skipped — only where the exclusion happens differs. // -// The client-side path also takes the MaxRows cap off the inner query. That -// store caps the rows it fetched, which the bump lets outnumber Limit, so +// The client-side path also takes the MaxRows cap off a limited inner query. +// That store caps the rows it fetched, which the bump lets outnumber Limit, so // `--limit N --max-rows N` failed with exit 2 as soon as more than N issues // were ready, refusing a page that could never exceed N. The caller enforces // the cap on the rows it keeps instead (capKeptRows), after the Limit trim, // which is where issueops enforces it on the rows a SQL store delivers // (finishReadyWorkWithCounts). +// +// An unlimited inner query keeps its cap, bumped the same way to MaxRows plus +// len(blockedIDs). Past that many rows, more than MaxRows of them are kept +// whichever rows the drop takes, so the bump refuses nothing the caller's cap +// would admit, and a store that sizes an unlimited request by MaxRows (the +// http bridge, under its client read cap) asks for that bound and one row +// more rather than the whole ready set. Its refusal reaches the caller as the +// caller's own (innerCapError). func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][]string) (queryFilter types.WorkFilter, blockedIDs map[string]bool) { if len(refsByIssue) == 0 { return filter, nil @@ -310,17 +319,21 @@ func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][ blockedIDs[issueID] = true } queryFilter = filter - if queryFilter.Limit > 0 { + switch { + case filter.Limit > 0: queryFilter.Limit += len(blockedIDs) + queryFilter.MaxRows = 0 + queryFilter.MaxRowsSource = "" + case filter.MaxRows > 0: + queryFilter.MaxRows += len(blockedIDs) } - queryFilter.MaxRows = 0 - queryFilter.MaxRowsSource = "" return queryFilter, blockedIDs } -// capKeptRows enforces the MaxRows cap readyExclusion took off the inner -// query, on the kept rows the caller is handed. With nothing to drop -// client-side the inner query kept the cap and has already enforced it. +// capKeptRows enforces the caller's MaxRows cap on the kept rows it is +// handed: readyExclusion took that cap off the inner query, or bumped it by +// the rows the caller drops. With nothing to drop client-side the inner query +// kept the cap and has already enforced it. func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) error { if len(blockedIDs) == 0 { return nil @@ -328,6 +341,22 @@ func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) return issueops.EnforceMaxRowsCap(kept, filter.MaxRows, filter.MaxRowsSource) } +// innerCapError hands the caller its own cap's refusal when the inner query +// refused the MaxRows readyExclusion bumped: Found rows, more than MaxRows +// plus len(blockedIDs), of which at least Found-len(blockedIDs), more than +// MaxRows, would have been kept. Any other error is returned as it is. +func innerCapError(err error, filter types.WorkFilter, blockedIDs map[string]bool) error { + var tooMany *issueops.ErrTooManyRows + if len(blockedIDs) == 0 || filter.Limit > 0 || filter.MaxRows <= 0 || !errors.As(err, &tooMany) { + return err + } + return &issueops.ErrTooManyRows{ + Found: tooMany.Found - len(blockedIDs), + Cap: filter.MaxRows, + Source: filter.MaxRowsSource, + } +} + // dropBlockedIssues is readyExclusion's client-side half: it drops the rows // blockedIDs names and trims what remains to limit. It copies into a fresh // slice rather than compacting issues in place, so the slice s.inner returned @@ -391,7 +420,7 @@ func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.Wo queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) rows, total, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, queryFilter) if err != nil { - return nil, 0, err + return nil, 0, innerCapError(err, filter, blockedIDs) } kept, blockedSeen := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) if len(blockedIDs) == 0 { @@ -407,14 +436,14 @@ func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.Wo // entire ready set, so every blocked row the window contains is every // blocked row the ready set contains: the subtraction is exact. This // is judged from the rows that came back, not from the Limit asked - // for: a zero Limit means every row to a SQL store, but the http wire - // omits it and the server answers with its default page - // (workapi.DefaultReadyLimit), so an unlimited fetch can come back - // truncated too. + // for, so it holds however a store reads a zero Limit: every row to a + // SQL store, and over http one request bounded at the client cap plus + // one, refused rather than truncated past it (an older http client + // sent no limit at all and got the server's default page back). adjustedTotal = total - blockedSeen } else { // The ready set is bigger than the window that came back (the bumped - // window, or the server's default page for an unlimited fetch), so + // window: the caller's page, or CountReadyWork's one-row page), so // the window cannot say how many of its unseen rows are blocked. Nor // can refsByIssue: it names every workspace issue with an unsatisfied // external blocker, closed ones and ones the filter excludes included, @@ -471,8 +500,16 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in } // s.inner has no indexed counter (httpclient.Store among others): reuse // the counts-and-total path above, which already copes with a store whose - // transport cannot express ExcludeIDs at all, nor this zero Limit. - _, total, err := s.readyWorkWithCountsAndTotal(ctx, filter, state) + // transport cannot express ExcludeIDs at all. It asks for a ONE-ROW Lite + // page, never every row: the total is the inner store's own count of the + // whole ready set (over http, the server's countReadyWork beside that + // page), and refetchedReadyTotal sizes the externally blocked case. A zero + // Limit would download the whole ready set to report one number — over + // http, up to BEADS_HTTP_UNLIMITED_READ_CAP full rows, refused past it. + page := filter + page.Limit = 1 + page.Lite = true + _, total, err := s.readyWorkWithCountsAndTotal(ctx, page, state) return total, err } diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index 3909275ed..c5eaf52c9 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -34,10 +34,15 @@ type fakeStore struct { excludeIDsUnsupported bool // defaultPage, when positive, is how many rows a zero-Limit ready listing // returns, while GetReadyWorkWithCountsAndTotal's total still counts the - // whole set: httpclient.Store's shape, whose wire omits a zero limit so - // the server answers with its default page. Zero keeps the SQL stores' - // meaning, every row. + // whole set: the shape httpclient.Store had while its wire omitted a zero + // limit and the server answered with its default page. That store now + // bounds an unlimited read itself, but the decorator still judges a window + // by the rows that came back, which this keeps pinned. Zero keeps the SQL + // stores' meaning, every row. defaultPage int + // readyFilters records every GetReadyWorkWithCountsAndTotal filter, in + // order. + readyFilters []types.WorkFilter } func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } @@ -124,6 +129,7 @@ func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.Wor } func (f *fakeStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + f.readyFilters = append(f.readyFilters, filter) items, err := f.GetReadyWorkWithCounts(ctx, filter) if err != nil { return nil, 0, err @@ -606,13 +612,64 @@ func TestReadyTotalsWhenStoreCannotExpressExcludeIDs(t *testing.T) { } } +// TestCountReadyWorkFallbackFetchesABoundedPage pins what CountReadyWork asks +// of a store with no ReadyWorkCounter: a one-row Lite page whose total is the +// store's own count, never a zero Limit. Over http a zero Limit is an +// unlimited read, which would download every ready row to report one number +// and be refused past BEADS_HTTP_UNLIMITED_READ_CAP. +func TestCountReadyWorkFallbackFetchesABoundedPage(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + for _, tc := range []struct { + name string + deps map[string][]*types.Dependency + want int + // wantFirst is the first fetch's Limit: the one-row page, bumped by + // every externally blocked id, since this store drops them itself. + wantFirst int + }{ + {name: "nothing externally blocked", want: 3, wantFirst: 1}, + { + name: "one issue externally blocked", + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + want: 2, + wantFirst: 2, + }, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &fakeStore{ready: []*types.Issue{a, b, c}, deps: tc.deps, excludeIDsUnsupported: true} + got, err := testStore(raw, &fakeStore{}, true).CountReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("CountReadyWork: %v", err) + } + if got != tc.want { + t.Fatalf("ready count = %d, want %d", got, tc.want) + } + if len(raw.readyFilters) == 0 { + t.Fatal("CountReadyWork fetched nothing; the fallback is not on the path") + } + if first := raw.readyFilters[0]; first.Limit != tc.wantFirst || !first.Lite { + t.Errorf("first fetch: Limit %d, Lite %v; want Limit %d, Lite true", first.Limit, first.Lite, tc.wantFirst) + } + for i, f := range raw.readyFilters { + if f.Limit <= 0 { + t.Errorf("fetch %d asked for every row (Limit %d)", i, f.Limit) + } + } + }) + } +} + // TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage pins the shape -// httpclient.Store has and an every-row Limit==0 fake hid: the wire omits a -// zero Limit, the server answers with its default page, and the total still -// counts the whole ready set. CountReadyWork, text-mode bd ready's total, -// always fetches with Limit 0, so be-z (ready, externally blocked, and past -// that page) once stayed in its total because the decorator took a zero Limit -// for a window that held every row. +// httpclient.Store once had and an every-row Limit==0 fake hid: the wire +// omitted a zero Limit, the server answered with its default page, and the +// total still counted the whole ready set. CountReadyWork, text-mode bd +// ready's total, fetched with Limit 0 then, so be-z (ready, externally +// blocked, and past that page) stayed in its total because the decorator took +// a zero Limit for a window that held every row. The count now asks for a +// one-row page and the JSON total below still asks with Limit 0: both must +// leave be-z out. func TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage(t *testing.T) { const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit ready := make([]*types.Issue, 0, defaultPage+2) diff --git a/issueops/querier.go b/issueops/querier.go index edd3b22b1..c02bf789b 100644 --- a/issueops/querier.go +++ b/issueops/querier.go @@ -61,16 +61,21 @@ type QueryRequest struct { // than a synonym for unlimited: two spellings of "no bound" is one more // than a caller can check for. // - // NOTHING BACKSTOPS IT. ListRequest pairs its Limit with MaxRows, a - // circuit breaker that refuses an oversized answer outright; this request - // has no twin to it by design, and neither knob that sets that one - // (`--max-rows`, BEADS_MAX_ROWS) is read anywhere on the query path. So an - // unlimited query is unbounded in the plain sense: it reads every row the - // expression matches, however many that is, and a caller who asked for one - // gets it rather than an ErrTooManyRows. The ceiling here is Limit itself. - // This is reachable without asking: a piped `bd query` resolves its - // unflagged limit to 0 before the request, the same policy `bd list` has - // applied to piped output since GH#4094. + // NOTHING BACKSTOPS IT on any backend but http (below). ListRequest pairs + // its Limit with MaxRows, a circuit breaker that refuses an oversized + // answer outright; this request has no twin to it by design, and neither + // knob that sets that one (`--max-rows`, BEADS_MAX_ROWS) is read anywhere + // on the query path. So an unlimited query is unbounded in the plain sense: + // it reads every row the expression matches, however many that is, and a + // caller who asked for one gets it rather than an ErrTooManyRows. The + // ceiling here is Limit itself. This is reachable without asking: a piped + // `bd query` resolves its unflagged limit to 0 before the request, the same + // policy `bd list` has applied to piped output since GH#4094. + // + // The http backend is the one exception, because its whole answer is one + // response: it bounds an unlimited query at BEADS_HTTP_UNLIMITED_READ_CAP + // rows and refuses a larger answer with httpclient.ErrUnlimitedReadCap + // rather than truncating it. Limit *int // Offset skips the first N MATCHING rows, on EVERY implementation and for diff --git a/issueops/reader.go b/issueops/reader.go index 3a7d3b0ce..dd761270c 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -93,7 +93,9 @@ type ReadyRequest struct { // Limit bounds the page. Nil means the shared ready default; 0 means // unlimited. It is a pointer so that "unset" and "explicitly unlimited" // stay distinguishable, which is what lets one constant serve both - // surfaces. + // surfaces. Over the http backend an unlimited ready read is one + // response, bounded at BEADS_HTTP_UNLIMITED_READ_CAP rows and refused + // past it with httpclient.ErrUnlimitedReadCap, never truncated. Limit *int // Offset skips the first N matching rows, on EVERY implementation. The // page a caller receives is the rows at positions [Offset, Offset+Limit) From 559b5de1e2be4b75e7cf0be12c48e444181d171b Mon Sep 17 00:00:00 2001 From: Stephanie Jarmak <36544495+sjarmak@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:28:54 +0000 Subject: [PATCH 08/99] fix(memory): preserve presence for empty values (#5964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(memory): preserve presence for empty values (#5963) Recall and Forget now distinguish an existing empty memory row from an absent key across direct, embedded, and UOW storage paths. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of sjarmak * fix(memory): make bd recall report presence from row existence printRecallResult (and the recallCmd caller) derived "found" from value != "", so a memory stored as the empty string still printed "No memory with key" and exited 1 even though the role's RecallResult.Found was true and bd forget would delete the row. Thread result.Found through instead: an empty-valued memory now recalls cleanly (exit 0, empty value; JSON reports found: true), and only an absent key is a miss. Addresses maintainer review on #5964 (fixes #5963). * test(httpapi): pin GET /v0/beads/memories/{key} as 200 for an empty value TestGetMemoryAnswersAMissWithA404 pinned a RecallResult{Value: ""} case (Found defaulting to its zero value, false) as a 404, a shape no backend can produce since Found now comes from row existence rather than value truthiness. Move that case into its own TestGetMemoryAnswersAnEmptyValueWith200, which recalls Found: true and asserts a 200 with an empty value. Also updates the handler's doc comment, which still described the retired 404-on-empty behavior. Addresses maintainer review on #5964 (fixes #5963). * docs(httpapi): fix openapi spec to say memory presence is row existence The published spec still said an empty-valued memory is a 404 on GET /v0/beads/memories/{key} and DELETE /v0/beads/memories/{key}, contradicting the role's RecallResult.Found (row existence). Rewrite the operation descriptions, both 404 response bodies, Memory.value and RememberedMemory.replaced to say presence is row existence everywhere: an empty-string memory is a 200 (or is removed by forget) with an empty value, matching what GET /v0/beads/memories already enumerated. Regenerate types.gen.go via `make api-gen` so the generated doc comments match. Addresses maintainer review on #5964 (fixes #5963). * docs(changelog): note the empty-valued memory presence fix Add an Unreleased/Fixed entry for #5964 (fixes #5963): bd recall, bd forget and GET /v0/beads/memories/{key} now treat an empty-valued memory as present everywhere, instead of only on GET /v0/beads/memories. * fix(memory): make bd remember recall an empty-valued memory rememberBareKeyPath decided presence from existing != "", so a bare slug naming a memory stored as the empty string was refused as "no memory named" instead of recalled. It now takes the role's Found, matching printRecallResult, bd forget and the HTTP door. * test(memory): pin empty-value presence through the real commands TestPrintRecallResult and TestRememberBareKeyPath call the helpers directly, so a RunE that went back to deciding presence from value != "" would leave them green. TestEmbeddedMemory/empty_valued_memory_is_present empties a memory row beneath bd and runs bd recall, bd remember and bd forget against it, checking first that bd no longer sees the placeholder. Two comments in problem.go and one in memories_test.go still said an empty-valued memory is a 404. The CHANGELOG line now says a key with no row is a miss, since a forgotten key was once stored. * build(bazel): add the memoryops dep to the bd_test go_test (make bazel-sync) memory_embedded_test.go now imports internal/storage/memoryops directly (storagememoryops.StorageKey, to overwrite a memory row out of band), so gazelle emits //internal/storage/memoryops on bd_test. cmd/bd/BUILD.bazel landed on main after this PR branched, so the PR could not carry the dep; without it, bazel.yml's "BUILD files in sync" step fails in the required bazel-test lane. Generated by `make bazel-sync`. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(memory): maintainer fixup — adoption review nits Maintainer fixup for the adopt-pr iteration-1 review of #5964. Both review lanes approved and the quality scorecard approved at 981; these are its four non-gating nits, each confined to lines this PR already adds. - internal/httpapi: TestForgetMemoryOfAnEmptyValuedKeyIs200 pins the DELETE door's empty-value 200 the way TestGetMemoryAnswersAnEmptyValueWith200 pins the read. Guarding the handler's 404 on Value == "" reddens exactly this test and nothing else. - CHANGELOG: the bullet names DELETE /v0/beads/memories/{key} beside GET, whose answer changed the same way (404 with the row left standing -> 200 with the row removed). - uow: the two new GetAllConfig presence probes in Recall and Forget say why they read the whole table, by pointing at Remember's rationale. - conformance: RunMemoriesEmptyRowIsPresentUntilForgotten runs its no-history-on-retry leg last and SKIPs it with a reason when a fixture has no CountHistory, as the MemoriesFixture doc promises, instead of passing quietly. All three in-tree fixtures wire the hook, so every in-tree run is unchanged (19/19 on dolt, embeddeddolt and uow, 0 SKIP). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * fix(httpclient): report an empty-valued memory as present (base-refresh repair) Maintainer repair from refreshing #5964 onto main b630f96f5b. Main's #7247 added the HTTP client store (internal/httpclient) after this PR branched, and its memory role and served conformance leg still treated a memory stored as the empty string as absent: - Recall set Found from Value != "". - The RecallMemory doc said an empty-valued row was a 404. - TestServedMemoriesRecallConflatesStoredEmptyWithAbsent called conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent, which this PR deletes. The rebase applied cleanly, but the merged tree did not compile. - internal/httpclient/memories.go: Recall reports Found true on any 200. With this PR the wire returns a 404 only for a key with no row; an empty-valued row comes back as a 200 with an empty value. Setting Found from Value would hide that row again on the HTTP backend. Forget already reported Found true on a 200. - internal/httpclient/wire/writes.go: the RecallMemory doc now says the same. - internal/httpclient/served_memories_deps_test.go: the hm10 served case now runs RunMemoriesEmptyRowIsPresentUntilForgotten, the contract case this PR adds in place of the one it deletes. With BEADS_TEST_EMBEDDED_DOLT=1 the served memory tests pass with 0 SKIP. Reverting the Recall line makes hm10 fail with `Recall("hm10-stored-empty") = value "" found false, want value "" found true`. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: sjarmak Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 15 ++ backend/conformance/memories_contract.go | 105 +++++++----- backend/conformance/role_bundle_cases.go | 2 +- cmd/bd/BUILD.bazel | 1 + cmd/bd/memory.go | 24 +-- cmd/bd/memory_embedded_test.go | 48 ++++++ cmd/bd/memory_pure_test.go | 158 +++++++++++++++++- internal/httpapi/apigen/types.gen.go | 4 +- internal/httpapi/memories.go | 9 +- internal/httpapi/memories_test.go | 110 ++++++++---- internal/httpapi/problem.go | 12 +- internal/httpapi/spec/openapi.v0.yaml | 49 +++--- internal/httpclient/memories.go | 7 +- .../httpclient/served_memories_deps_test.go | 4 +- internal/httpclient/wire/writes.go | 8 +- internal/storage/dolt/memories.go | 9 +- .../storage/dolt/memories_contract_test.go | 4 +- internal/storage/embeddeddolt/memories.go | 9 +- .../embeddeddolt/memories_contract_test.go | 4 +- internal/storage/memoryops/memories.go | 30 ++-- internal/storage/uow/memories.go | 16 +- .../storage/uow/memories_contract_test.go | 4 +- memoryops/errors.go | 15 +- memoryops/memories.go | 25 +-- 24 files changed, 476 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eea684e1..1ab3cbdec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -795,6 +795,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 route; the direct and embedded routes are unchanged ([#5861](https://github.com/gastownhall/beads/issues/5861) item 1). +- **`bd recall`, `bd forget`, `bd remember `, + `GET /v0/beads/memories/{key}` and `DELETE /v0/beads/memories/{key}` now + treat an empty-valued memory as present** + ([#5964](https://github.com/gastownhall/beads/pull/5964), fixes + [#5963](https://github.com/gastownhall/beads/issues/5963)). Presence used to + be derived from the stored value's truthiness, so a memory written out of + band with an empty string was invisible even though its row existed. `bd + recall` printed "No memory with key" and exited 1, `bd forget` refused to + delete it, `bd remember ` refused to recall it, and both HTTP + doors answered 404 (the DELETE leaving the row in place), while + `GET /v0/beads/memories` already enumerated the same row. Presence now comes from row existence on direct Dolt, embedded Dolt and + the proxied backend. An empty-valued memory is found, recalls as an empty + string, and is deleted on the first forget. Only a key with no row is a + miss. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/backend/conformance/memories_contract.go b/backend/conformance/memories_contract.go index c7b33221c..dc45191f4 100644 --- a/backend/conformance/memories_contract.go +++ b/backend/conformance/memories_contract.go @@ -233,14 +233,11 @@ func RunMemoriesRememberWithExplicitKeyStoresVerbatim(t *testing.T, ctx context. } // RunMemoriesRememberReplacesAndReportsIt pins -// memoryops.RememberResult.Replaced, including the leg that makes it different -// from RecallResult.Found. +// memoryops.RememberResult.Replaced against row presence. // // Three legs: a first write reports false, a second reports true and REPLACES -// rather than appends, and a write over an out-of-band EMPTY row reports true -// even though a Recall of that key would have reported Found false. The third -// is the divergence the leaf doc states — Replaced is about the ROW — and it is -// the one a body computing `previous != ""` gets wrong. +// rather than appends, and a write over an out-of-band EMPTY row reports true. +// The third is the one a body computing `previous != ""` gets wrong. func RunMemoriesRememberReplacesAndReportsIt(t *testing.T, ctx context.Context, fixture MemoriesFixture) { t.Helper() neighbors := seedMemoriesAllFourClasses(t, ctx, fixture) @@ -262,7 +259,7 @@ func RunMemoriesRememberReplacesAndReportsIt(t *testing.T, ctx context.Context, if err := fixture.SetConfig(ctx, "kv.memory."+emptied, ""); err != nil { t.Fatalf("seed an empty memory row out of band: %v", err) } - assertMemoriesRecall(t, ctx, fixture, emptied, "", false) + assertMemoriesRecall(t, ctx, fixture, emptied, "", true) overEmpty := rememberMemory(t, ctx, fixture, memoryops.RememberRequest{Key: emptied, Content: "now it has content"}) if !overEmpty.Replaced { t.Fatalf("Remember over a row stored EMPTY reported Replaced false: Replaced is about the row, " + @@ -374,8 +371,8 @@ func RunMemoriesRecallAnswersTheStoredValue(t *testing.T, ctx context.Context, f } // RunMemoriesRecallReportsAMissAsNotFoundNotAnError pins the decision recorded -// in memoryops/errors.go: there is no ErrNotFound on this role, because the -// seam beneath it cannot see the difference the error would claim. +// in memoryops/errors.go: absence is a result callers inspect, not an +// operational error they must classify. func RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T, ctx context.Context, fixture MemoriesFixture) { t.Helper() seedMemoriesAllFourClasses(t, ctx, fixture) @@ -401,33 +398,70 @@ func RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T, ctx context } } -// RunMemoriesRecallConflatesStoredEmptyWithAbsent pins -// memoryops.RecallResult.Found's central claim, and the one asymmetry that lets -// a caller see past it: List enumerates the key, because the ROW exists. +// RunMemoriesEmptyRowIsPresentUntilForgotten pins presence to the ROW rather +// than to whether its value is non-empty. No front door creates an empty +// memory, but out-of-band config writes and older workspaces can contain one, +// and it must remain addressable through Recall and Forget. // -// That is the same same-answer-different-row shape the settings contract pins -// for `bd config get`, and stating it is what keeps a backend from "fixing" it -// on one leg and making the answer depend on the route. -func RunMemoriesRecallConflatesStoredEmptyWithAbsent(t *testing.T, ctx context.Context, fixture MemoriesFixture) { +// The second Forget is the retry leg: after the first call deletes the row, it +// reports a genuine miss and records no further history. +func RunMemoriesEmptyRowIsPresentUntilForgotten(t *testing.T, ctx context.Context, fixture MemoriesFixture) { t.Helper() - seedMemoriesAllFourClasses(t, ctx, fixture) + neighbors := seedMemoriesAllFourClasses(t, ctx, fixture) emptied := fixture.IssuePrefix + "-stored-empty" never := fixture.IssuePrefix + "-never-stored" if err := fixture.SetConfig(ctx, "kv.memory."+emptied, ""); err != nil { t.Fatalf("seed an empty memory row out of band: %v", err) } - assertMemoriesRecall(t, ctx, fixture, emptied, "", false) + assertMemoriesRecall(t, ctx, fixture, emptied, "", true) assertMemoriesRecall(t, ctx, fixture, never, "", false) plane := listMemories(t, ctx, fixture, "") if value, ok := plane[emptied]; !ok || value != "" { - t.Fatalf("List[%q] = %q (present=%v), want an empty value that IS enumerated: "+ - "the row exists even though Recall denies it", emptied, value, ok) + t.Fatalf("List[%q] = %q (present=%v), want an enumerated empty value", emptied, value, ok) } if _, ok := plane[never]; ok { t.Fatalf("List carries %q, which nothing stored", never) } + + first := forgetMemory(t, ctx, fixture, emptied) + if !first.Found || first.Value != "" { + t.Fatalf("first Forget(%q) = %+v, want Found true with the empty stored value", emptied, first) + } + assertMemoriesRawAbsent(t, ctx, fixture, "kv.memory."+emptied) + assertMemoriesNeighborsSurvived(t, ctx, fixture, neighbors) + + var afterFirst int + if fixture.CountHistory != nil { + var err error + afterFirst, err = fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory after first Forget: %v", err) + } + } + + second := forgetMemory(t, ctx, fixture, emptied) + if second.Found || second.Value != "" { + t.Fatalf("second Forget(%q) = %+v, want Found false with no value", emptied, second) + } + assertMemoriesRawAbsent(t, ctx, fixture, "kv.memory."+emptied) + assertMemoriesNeighborsSurvived(t, ctx, fixture, neighbors) + + // The no-write half of the retry is the only leg that needs history, which + // is why it comes last: a backend that cannot observe history has run every + // other leg by here, and SKIPS this one with that reason, as the + // MemoriesFixture.CountHistory doc promises, rather than passing quietly. + if fixture.CountHistory == nil { + t.Skip("this backend cannot observe history, so the no-write half of the retry is unobservable here") + } + afterSecond, err := fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory after second Forget: %v", err) + } + if afterSecond != afterFirst { + t.Fatalf("history entries went %d -> %d across the second Forget, want no write", afterFirst, afterSecond) + } } // RunMemoriesForgetRemovesExactlyTheNamedRow IS THE QUADRANT CASE, and the @@ -528,13 +562,7 @@ func RunMemoriesForgetReportsTheForgottenValue(t *testing.T, ctx context.Context // RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing pins that a miss is // a RESULT and not an error, that it changes no row anywhere in the table, that // the second call — the one a retrying caller actually makes — answers the same -// way, and that the SECOND kind of miss behaves like the first: a memory stored -// as the empty string is Found false, and its row is left standing. -// -// That last leg is memoryops.ForgetResult.Found's parenthesis, and it keeps -// Forget's answer and Recall's answer the same statement. A body that deleted -// on "the row is there" rather than on "Recall would find it" would report -// nothing removed while removing something. +// way. // // The whole-table row count is the assertion that matters: "the key is still // absent" would pass on a delete that swept a neighboring plane. @@ -542,10 +570,6 @@ func RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T, ctx t.Helper() neighbors := seedMemoriesAllFourClasses(t, ctx, fixture) key := fixture.IssuePrefix + "-never-stored-forget" - emptied := fixture.IssuePrefix + "-empty-forget" - if err := fixture.SetConfig(ctx, "kv.memory."+emptied, ""); err != nil { - t.Fatalf("seed an empty memory row out of band: %v", err) - } var before int if err := fixture.QueryScalar(ctx, "SELECT COUNT(*) FROM config", nil, &before); err != nil { @@ -553,14 +577,12 @@ func RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T, ctx } for attempt := 1; attempt <= 2; attempt++ { - for _, miss := range []string{key, emptied} { - result, err := fixture.Memories.Forget(ctx, memoryops.ForgetRequest{Key: miss}) - if err != nil { - t.Fatalf("Forget(%q) attempt %d = %v, want success with Found false", miss, attempt, err) - } - if result.Found || result.Value != "" { - t.Fatalf("Forget(%q) attempt %d = %+v, want Found false and no value", miss, attempt, result) - } + result, err := fixture.Memories.Forget(ctx, memoryops.ForgetRequest{Key: key}) + if err != nil { + t.Fatalf("Forget(%q) attempt %d = %v, want success with Found false", key, attempt, err) + } + if result.Found || result.Value != "" { + t.Fatalf("Forget(%q) attempt %d = %+v, want Found false and no value", key, attempt, result) } } @@ -569,11 +591,8 @@ func RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T, ctx t.Fatalf("count config rows after: %v", err) } if after != before { - t.Fatalf("config rows went %d -> %d across four forgets that found nothing, want no change", before, after) + t.Fatalf("config rows went %d -> %d across two forgets that found nothing, want no change", before, after) } - // The stored-empty row is specifically still there: Found false said nothing - // was removed, and that has to be true of the row as well as of the answer. - assertMemoriesRawValue(t, ctx, fixture, "kv.memory."+emptied, "") assertMemoriesNeighborsSurvived(t, ctx, fixture, neighbors) // The empty key is a refusal, not a miss. diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 782a20306..36c158477 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -505,7 +505,7 @@ var roleContractCases = []roleContract{ RunMemoriesRememberRefusesAnUnderivableKey, RunMemoriesRecallAnswersTheStoredValue, RunMemoriesRecallReportsAMissAsNotFoundNotAnError, - RunMemoriesRecallConflatesStoredEmptyWithAbsent, + RunMemoriesEmptyRowIsPresentUntilForgotten, RunMemoriesForgetRemovesExactlyTheNamedRow, RunMemoriesForgetNeverTouchesTheSettingsPlane, RunMemoriesForgetReportsTheForgottenValue, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6506f2bd3..7c493dc27 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1350,6 +1350,7 @@ go_test( "//internal/storage/issueops", "//internal/storage/journalscan", "//internal/storage/kvkeys", + "//internal/storage/memoryops", "//internal/storage/schema", "//internal/storage/uow", "//internal/storage/versioncontrolops", diff --git a/cmd/bd/memory.go b/cmd/bd/memory.go index 944ec014a..1f0746b05 100644 --- a/cmd/bd/memory.go +++ b/cmd/bd/memory.go @@ -217,9 +217,11 @@ func rememberBudgetVerdict(existing map[string]string, key, content string, budg // `bd remember ` (no --key): a bare slug naming an EXISTING memory // is recalled instead of stored; a bare slug naming nothing is refused. The // caller only invokes it when memoryKeyFlag == "" and the insight round-trips -// through memoryapi.DeriveKey unchanged, having already read the key. -func rememberBareKeyPath(key, insight, existing string) error { - if existing != "" { +// through memoryapi.DeriveKey unchanged, having already read the key. As in +// printRecallResult, presence is found (row existence), never existing != "": +// a bare slug naming a memory stored as the empty string recalls it. +func rememberBareKeyPath(key, insight, existing string, found bool) error { + if found { if jsonOutput { return outputJSON(map[string]interface{}{ "key": key, @@ -317,22 +319,24 @@ func printForgetResult(key, existing string) error { } // printRecallResult renders the `bd recall` output (including the not-found -// SilentExit contract). -func printRecallResult(key, value string) error { +// SilentExit contract). Presence comes from found — ROW EXISTENCE, per the +// role's RecallResult.Found — never from value != "": a memory stored as the +// empty string is found and recalls as "", and only an absent key is a miss. +func printRecallResult(key, value string, found bool) error { if jsonOutput { if jerr := outputJSON(map[string]interface{}{ "key": key, "value": value, - "found": value != "", + "found": found, }); jerr != nil { return jerr } - if value == "" { + if !found { return SilentExit() } return nil } - if value == "" { + if !found { fmt.Fprintf(os.Stderr, "No memory with key %q\n", key) return SilentExit() } @@ -436,7 +440,7 @@ Examples: if err != nil { return HandleErrorRespectJSON("recalling memory: %v", err) } - return rememberBareKeyPath(derived, insight, recalled.Value) + return rememberBareKeyPath(derived, insight, recalled.Value, recalled.Found) } // Corpus budget (memories.budget-chars, OPT-IN, 0 = off). The @@ -653,7 +657,7 @@ Examples: return HandleErrorRespectJSON("recalling memory: %v", err) } - return printRecallResult(result.Key, result.Value) + return printRecallResult(result.Key, result.Value, result.Found) }, } diff --git a/cmd/bd/memory_embedded_test.go b/cmd/bd/memory_embedded_test.go index c39735030..21449412e 100644 --- a/cmd/bd/memory_embedded_test.go +++ b/cmd/bd/memory_embedded_test.go @@ -16,6 +16,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + storagememoryops "github.com/steveyegge/beads/internal/storage/memoryops" ) // countMemoryDoltCommits reads dolt_log for an embedded workspace. It is how @@ -41,6 +42,30 @@ func countMemoryDoltCommits(t *testing.T, beadsDir string) int { return count } +// storeMemoryValueOutOfBand overwrites a memory row's value with SQL, beneath +// every bd code path. It is the only way a test gets an empty-valued memory: +// `bd remember` refuses empty content, so such a row always arrives out of band. +func storeMemoryValueOutOfBand(t *testing.T, beadsDir, key, value string) { + t.Helper() + cfg, _ := configfile.Load(beadsDir) + database := "" + if cfg != nil { + database = cfg.GetDoltDatabase() + } + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer cleanup() + res, err := db.ExecContext(t.Context(), "UPDATE config SET value = ? WHERE `key` = ?", value, storagememoryops.StorageKey(key)) + if err != nil { + t.Fatalf("update memory %q: %v", key, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("update memory %q: %d rows affected (err %v), want 1", key, n, err) + } +} + // bdRemember runs "bd remember" with the given args and returns stdout. func bdRemember(t *testing.T, bd, dir string, args ...string) string { t.Helper() @@ -283,6 +308,29 @@ func TestEmbeddedMemory(t *testing.T) { bdRecallFail(t, bd, dir, "forget-me") }) + // AN EMPTY-VALUED MEMORY IS PRESENT (#5963). These run the real commands, so + // they pin each RunE's wiring and not only its renderer: a call site that + // went back to deciding presence from value != "" fails here while + // TestPrintRecallResult and TestRememberBareKeyPath stay green. Its own + // workspace, because the row is emptied beneath bd. + t.Run("empty_valued_memory_is_present", func(t *testing.T) { + edir, ebeads, _ := bdInit(t, bd, "--prefix", "em") + bdRemember(t, bd, edir, "placeholder the test empties", "--key", "stored-empty") + storeMemoryValueOutOfBand(t, ebeads, "stored-empty", "") + + // The placeholder check proves bd sees the emptied row. Without it, a + // write bd never read would leave this test green on a reverted fix. + if out := bdRecall(t, bd, edir, "stored-empty"); strings.Contains(out, "placeholder") { + t.Fatalf("bd recall still sees the placeholder, so the row was not emptied: %s", out) + } + // A bare key naming the empty-valued memory recalls it instead of refusing. + if out := bdRemember(t, bd, edir, "stored-empty"); strings.Contains(out, "placeholder") { + t.Fatalf("bd remember still sees the placeholder: %s", out) + } + bdForget(t, bd, edir, "stored-empty") + bdRecallFail(t, bd, edir, "stored-empty") + }) + // THE AUTO-COMMIT EPILOGUE, which is the trap this convergence was most // likely to spring. A direct-route write lands in the Dolt WORKING SET and // nothing else commits it: durability is the epilogue in main.go, keyed on diff --git a/cmd/bd/memory_pure_test.go b/cmd/bd/memory_pure_test.go index 5cd96fa82..6ad443c19 100644 --- a/cmd/bd/memory_pure_test.go +++ b/cmd/bd/memory_pure_test.go @@ -3,7 +3,11 @@ package main -import "testing" +import ( + "encoding/json" + "strings" + "testing" +) // TestMatchesKnownCommand guards the `bd remember ` misfire fix // (GH#4401): a bare word that names a real top-level command must be flagged so @@ -39,3 +43,155 @@ func TestMatchesKnownCommand(t *testing.T) { }) } } + +// TestPrintRecallResult guards the bd-5963 fix: presence must come from +// `RecallResult.Found` (row existence), never from `value != ""`. Before the +// fix, `bd recall` on a memory stored as the empty string printed "No memory +// with key" and exited 1 even though the row was there and `bd forget` would +// delete it — the CLI disagreed with the role and with the HTTP door about +// what "found" means for that row. +func TestPrintRecallResult(t *testing.T) { + t.Run("found with empty value prints the empty value and exits clean", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = false + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("empty-key", "", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out != "\n" { + t.Errorf("stdout = %q, want a single newline (the empty value)", out) + } + }) + + t.Run("found with empty value in JSON reports found true", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = true + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("empty-key", "", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var body map[string]interface{} + if uerr := json.Unmarshal([]byte(strings.TrimSpace(out)), &body); uerr != nil { + t.Fatalf("unmarshal %q: %v", out, uerr) + } + if body["found"] != true { + t.Errorf("found = %v, want true", body["found"]) + } + if body["value"] != "" { + t.Errorf("value = %v, want \"\"", body["value"]) + } + if body["key"] != "empty-key" { + t.Errorf("key = %v, want %q", body["key"], "empty-key") + } + }) + + t.Run("found with non-empty value is unchanged", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = false + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("k", "hello", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out != "hello\n" { + t.Errorf("stdout = %q, want %q", out, "hello\n") + } + }) + + t.Run("not found prints the miss and exits 1", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = false + defer func() { jsonOutput = savedJSONOutput }() + + var err error + stderr := captureStderr(t, func() { + err = printRecallResult("gone", "", false) + }) + if !strings.Contains(stderr, `No memory with key "gone"`) { + t.Errorf("stderr = %q, want it to name the missing key", stderr) + } + ee, ok := err.(*exitError) + if !ok || ee.Code != 1 { + t.Errorf("err = %v, want *exitError{Code: 1}", err) + } + }) + + t.Run("not found in JSON reports found false and exits 1", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = true + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("gone", "", false) + return nil + }) + var body map[string]interface{} + if uerr := json.Unmarshal([]byte(strings.TrimSpace(out)), &body); uerr != nil { + t.Fatalf("unmarshal %q: %v", out, uerr) + } + if body["found"] != false { + t.Errorf("found = %v, want false", body["found"]) + } + ee, ok := err.(*exitError) + if !ok || ee.Code != 1 { + t.Errorf("err = %v, want *exitError{Code: 1}", err) + } + }) +} + +// TestRememberBareKeyPath guards the same presence rule on the +// `bd remember ` read path: a bare slug naming a memory stored as +// the empty string recalls it, and only a slug naming nothing is refused. +func TestRememberBareKeyPath(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = true + defer func() { jsonOutput = savedJSONOutput }() + + t.Run("found with empty value recalls it", func(t *testing.T) { + var err error + out := captureStdout(t, func() error { + err = rememberBareKeyPath("empty-key", "empty-key", "", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var body map[string]interface{} + if uerr := json.Unmarshal([]byte(strings.TrimSpace(out)), &body); uerr != nil { + t.Fatalf("unmarshal %q: %v", out, uerr) + } + if body["action"] != "recalled" || body["found"] != true || body["value"] != "" { + t.Errorf("body = %v, want action recalled, found true, empty value", body) + } + }) + + t.Run("not found is refused", func(t *testing.T) { + var err error + captureStdout(t, func() error { + err = rememberBareKeyPath("gone", "gone", "", false) + return nil + }) + ee, ok := err.(*exitError) + if !ok || ee.Code != 1 { + t.Errorf("err = %v, want *exitError{Code: 1}", err) + } + }) +} diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index ae84aeca7..afd6eedff 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1563,7 +1563,7 @@ type Memory struct { // Value The stored content, verbatim: newlines, surrounding space and unicode as stored, never truncated and never withheld. // - // Always present. It is the empty string only where a row was written out of band with an empty value, which `GET /v0/beads/memories/{key}` answers as a `404` and `GET /v0/beads/memories` enumerates. + // Always present. It is the empty string only where a row was written out of band with an empty value — a row `GET /v0/beads/memories/{key}` answers `200` for, and `GET /v0/beads/memories` enumerates, the same as any other memory. Value string `json:"value"` } @@ -1865,7 +1865,7 @@ type RememberedMemory struct { // Replaced True when a previous value existed under `key` and this request overwrote it; false when the key was new. It is observed in the same transaction as the write, so it describes the row this request wrote. // - // A previous value that was the EMPTY STRING reports true: the ROW existed, even though `GET /v0/beads/memories/{key}` would have answered `404` for it. That divergence is the storage seam's conflation showing through, and it is stated rather than smoothed over, because smoothing it would mean this member reporting "nothing was there" about a write that overwrote something. + // A previous value that was the EMPTY STRING reports true: the ROW existed, and `GET /v0/beads/memories/{key}` would have answered `200` for it too, with an empty `value` — this member and that endpoint agree on what counts as present, because both read it off the row rather than off the value it happened to hold. Replaced bool `json:"replaced"` // Value The stored content, echoed verbatim. Always present, and never withheld — this plane has no redaction; see the operation description. diff --git a/internal/httpapi/memories.go b/internal/httpapi/memories.go index 2bf3271c0..06970a2e2 100644 --- a/internal/httpapi/memories.go +++ b/internal/httpapi/memories.go @@ -162,11 +162,10 @@ func (s *Server) handleGetMemory(w http.ResponseWriter, r *http.Request) { // SilentExit), and the role answers Found rather than a bare value, so the // status here reports a distinction that exists rather than inventing one. // - // A row stored as the EMPTY STRING falls on the miss side of it, because - // that is where the role puts it — the storage seam cannot tell it from an - // absent row, and this handler will not claim to see what the role cannot. - // listMemories enumerates such a row, which is the one way a client tells - // the two apart. + // Found is ROW EXISTENCE, not value truthiness. A row stored as the EMPTY + // STRING is found: it comes back as a 200 with an empty `value`, exactly + // like `GET /v0/beads/memories` already enumerates it. Only a key nothing + // stored is a miss. if !result.Found { s.fail(w, r, MemoryNotFound()) return diff --git a/internal/httpapi/memories_test.go b/internal/httpapi/memories_test.go index ee9636ae9..fafcc6ca0 100644 --- a/internal/httpapi/memories_test.go +++ b/internal/httpapi/memories_test.go @@ -279,41 +279,54 @@ func TestGetMemoryAnswersTheStoredValue(t *testing.T) { } // TestGetMemoryAnswersAMissWithA404 is the deliberate divergence from the -// settings surface's no-404 doctrine. -// -// Both legs are the SAME answer from the role — Found false — and both are a -// 404: a row stored as the empty string is a miss here because the storage seam -// cannot tell it from an absent row, and the wire does not claim to see what -// the role cannot. `GET /v0/beads/memories` enumerating such a row is the one -// way a client tells them apart. +// settings surface's no-404 doctrine: a key nothing stored is Found false from +// the role, and the wire reports a 404 rather than the settings plane's silent +// "(not set)" 200. func TestGetMemoryAnswersAMissWithA404(t *testing.T) { - for _, tc := range []struct { - name string - result memoryops.RecallResult - }{ - {name: "nothing stored", result: memoryops.RecallResult{Key: "gone"}}, - {name: "stored as the empty string", result: memoryops.RecallResult{Key: "gone", Value: ""}}, - } { - t.Run(tc.name, func(t *testing.T) { - ts := newTestServer(t, rolesConfig(Config{Memories: &roleMemories{recalled: tc.result}})) + ts := newTestServer(t, rolesConfig(Config{Memories: &roleMemories{ + recalled: memoryops.RecallResult{Key: "gone"}, + }})) - resp := ts.get(t, memoriesPath+"/gone") - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d, want 404: %s", resp.StatusCode, readAll(t, resp)) - } - body := decodeBody(t, resp) - if got := body["code"]; got != string(CodeNotFound) { - t.Errorf("code = %v, want %q", got, CodeNotFound) - } - if body["request_id"] == nil { - t.Error("no request_id on the problem body") - } - // The detail is about the MEMORY plane. Reusing the issue-id - // sentence would tell a client its memory key was an issue id. - if got, _ := body["detail"].(string); !strings.Contains(got, "memory") { - t.Errorf("detail = %q, want it to name the memory plane", got) - } - }) + resp := ts.get(t, memoriesPath+"/gone") + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if got := body["code"]; got != string(CodeNotFound) { + t.Errorf("code = %v, want %q", got, CodeNotFound) + } + if body["request_id"] == nil { + t.Error("no request_id on the problem body") + } + // The detail is about the MEMORY plane. Reusing the issue-id sentence + // would tell a client its memory key was an issue id. + if got, _ := body["detail"].(string); !strings.Contains(got, "memory") { + t.Errorf("detail = %q, want it to name the memory plane", got) + } +} + +// TestGetMemoryAnswersAnEmptyValueWith200 pins bd-5963's fix: presence comes +// from the role's Found (row existence), never from Value != "". A row +// written out of band with an empty value is a real row — the same one +// `GET /v0/beads/memories` already enumerates — so this door answers 200 with +// an empty value rather than the 404 it used to share with an absent key. +func TestGetMemoryAnswersAnEmptyValueWith200(t *testing.T) { + memories := &roleMemories{recalled: memoryops.RecallResult{ + Key: "gone", Value: "", Found: true, + }} + ts := newTestServer(t, rolesConfig(Config{Memories: memories})) + + resp := ts.get(t, memoriesPath+"/gone") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + + body := decodeBody(t, resp) + if got := body["key"]; got != "gone" { + t.Errorf("key = %v, want %q", got, "gone") + } + if got := body["value"]; got != "" { + t.Errorf("value = %v, want \"\"", got) } } @@ -440,6 +453,32 @@ func TestForgetMemoryOfAnAbsentKeyIs404AndRemovesNothing(t *testing.T) { } } +// TestForgetMemoryOfAnEmptyValuedKeyIs200 is the DELETE half of bd-5963's fix, +// pinned at the door as TestGetMemoryAnswersAnEmptyValueWith200 pins the read: +// a row stored as the empty string is Found, so it was removed, and the answer +// is a 200 carrying the empty value it held. A door that keyed its 404 on +// Value == "" would tell a client "there was nothing to remove" about a row it +// had just deleted, and the role's conformance contract could not see it. +func TestForgetMemoryOfAnEmptyValuedKeyIs200(t *testing.T) { + memories := &roleMemories{forgotten: memoryops.ForgetResult{ + Key: "stored-empty", Value: "", Found: true, + }} + ts := newTestServer(t, rolesConfig(Config{Memories: memories})) + + resp := ts.forget(t, memoriesPath+"/stored-empty") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + + body := decodeBody(t, resp) + if got := body["key"]; got != "stored-empty" { + t.Errorf("key = %v, want %q", got, "stored-empty") + } + if got := body["value"]; got != "" { + t.Errorf("value = %v, want \"\"", got) + } +} + // TestForgetMemoryRefusesAKeyItCannotAddress: the door refuses the same keys // the read refuses, and the role is never reached — which on a DESTRUCTIVE // operation is the half that matters. A stored key carrying a control @@ -526,9 +565,8 @@ func TestListMemoriesAnswersThePlaneOrderedByKey(t *testing.T) { "alpha": "first by key", "Has Spaces.✓": "an explicit key", // A row written out of band with an empty value. It is enumerated here - // — its KEY exists — while GET /v0/beads/memories/{key} answers 404 for - // it, and that asymmetry is the one way a client tells a row stored - // empty from a row that is not there. + // like any other row, and GET /v0/beads/memories/{key} answers it with a + // 200 and an empty value (TestGetMemoryAnswersAnEmptyValueWith200). "stored-empty": "", }}} ts := newTestServer(t, rolesConfig(Config{Memories: memories})) diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 2bad156a1..fc1a38b4a 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -1036,8 +1036,9 @@ var operationCodes = map[string][]Code{ // the CLI itself prints identically, so a 404 would publish an invented // distinction. Here the CLI already distinguishes a miss — `bd recall` has // an exit-code contract for it — and the role answers Found rather than a - // value, so the 404 reports a distinction that exists. The stored-empty row - // falls on the miss side of it, which the document states. + // value, so the 404 reports a distinction that exists. Found is row + // existence: a row stored as the empty string is a 200 with an empty value, + // and only an absent row is the 404, which the document states. OpGetMemory: {CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodeBusy, CodeDBUnavailable, CodeInternal}, // The read's vocabulary exactly, because the two operations address the same // resource the same way and this one's Found false is the same answer: a key @@ -1510,10 +1511,9 @@ func EventsWatchSaturated() Result { // // A separate constructor rather than a detail argument on NotFound, because the // two say different things: that one is about the issue id space, and reusing -// its sentence here would tell a client its memory key was an issue id. The -// detail deliberately does NOT distinguish an absent row from one stored as the -// empty string — the role cannot see the difference, so the wire must not claim -// to. +// its sentence here would tell a client its memory key was an issue id. It is +// only ever the answer for an absent row: a row stored as the empty string is +// found, and the read answers it with a 200. func MemoryNotFound() Result { return newResult(CodeNotFound, "this workspace holds no memory under that key") } diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index b77bfab1a..d95723cb0 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -5768,10 +5768,10 @@ paths: A memory stored as the EMPTY STRING is enumerated here, because its key - exists, while `GET /v0/beads/memories/{key}` answers `404` for it. That - asymmetry is the one way a client can tell a row stored empty from a row - that is not there, and it is the storage seam's conflation showing - through rather than a rule this surface invented. + exists, and `GET /v0/beads/memories/{key}` agrees: it answers `200` for + that same row, with an empty `value`. Presence is row existence on both + surfaces, never whether the value happens to be non-empty; only a key + nothing stored is missing from either one. The envelope is the paginated one and `has_more` is ALWAYS false: @@ -5928,12 +5928,13 @@ paths: — so a 404 reports a distinction that exists rather than minting one. - A MEMORY STORED AS THE EMPTY STRING IS A 404 TOO. No front door can - create one — storing empty content is refused — but an out-of-band write - to the config table can, and the role does not invent a distinction the - storage seam beneath it cannot see. `GET /v0/beads/memories` DOES - enumerate such a row, because its key exists, and that asymmetry is the - one way a client can tell the two apart. + A MEMORY STORED AS THE EMPTY STRING IS A 200, WITH AN EMPTY `value`. No + front door can create one — storing empty content is refused — but an + out-of-band write to the config table can, and presence here is row + existence, never whether the value happens to be non-empty. That is the + same rule `GET /v0/beads/memories` already followed by enumerating such + a row because its key exists; the two surfaces now agree rather than + splitting on an asymmetry a client would have had to know about. MEMORY CONTENT IS SERVED IN FULL, and a configured bearer does not @@ -5975,9 +5976,9 @@ paths: $ref: '#/components/responses/Unauthenticated' '404': description: >- - This workspace holds no memory under that key — or holds one stored - as the empty string, which answers the same way. `detail` says which - plane was asked, not which of those two it was. + This workspace holds no memory under that key. A row stored as the + empty string is not this case: it exists, so this operation returns + it as a `200` with an empty `value`, the same as any other memory. x-bd-codes: [not_found] content: application/problem+json: @@ -6059,9 +6060,10 @@ paths: $ref: '#/components/responses/Unauthenticated' '404': description: >- - This workspace holds no memory under that key — or holds one stored - as the empty string, which answers the same way — and NOTHING WAS - REMOVED. + This workspace holds no memory under that key, and NOTHING WAS + REMOVED. A row stored as the empty string is not this case: it + exists, so forgetting it removes it and answers `200`, the same as + any other memory. x-bd-codes: [not_found] content: application/problem+json: @@ -9355,9 +9357,9 @@ components: Always present. It is the empty string only where a row was written - out of band with an empty value, which - `GET /v0/beads/memories/{key}` answers as a `404` and - `GET /v0/beads/memories` enumerates. + out of band with an empty value — a row `GET /v0/beads/memories/{key}` + answers `200` for, and `GET /v0/beads/memories` enumerates, the same + as any other memory. MemoriesPage: type: object @@ -9445,11 +9447,10 @@ components: A previous value that was the EMPTY STRING reports true: the ROW - existed, even though `GET /v0/beads/memories/{key}` would have - answered `404` for it. That divergence is the storage seam's - conflation showing through, and it is stated rather than smoothed - over, because smoothing it would mean this member reporting - "nothing was there" about a write that overwrote something. + existed, and `GET /v0/beads/memories/{key}` would have answered + `200` for it too, with an empty `value` — this member and that + endpoint agree on what counts as present, because both read it off + the row rather than off the value it happened to hold. SetSettingRequest: type: object diff --git a/internal/httpclient/memories.go b/internal/httpclient/memories.go index e5bc3f655..986f6a8f4 100644 --- a/internal/httpclient/memories.go +++ b/internal/httpclient/memories.go @@ -66,9 +66,10 @@ func (m *httpMemories) Recall(ctx context.Context, req memoryops.RecallRequest) if err != nil { return memoryops.RecallResult{}, err } - // Found is Value != "": the role conflates a stored empty string with an - // absent memory, and so does the wire, which 404s the same row. - return memoryops.RecallResult{Key: req.Key, Value: res.Value, Found: res.Value != ""}, nil + // Found is about the ROW, not the value: the wire spells a miss as the 404 + // handled above, and a row stored as the empty string as a 200 with an + // empty value. Reading Found off Value would hide that row again. + return memoryops.RecallResult{Key: req.Key, Value: res.Value, Found: true}, nil } // Forget removes one memory and reports the value it removed. diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go index 5550f0b4b..d73b741b0 100644 --- a/internal/httpclient/served_memories_deps_test.go +++ b/internal/httpclient/served_memories_deps_test.go @@ -80,8 +80,8 @@ func TestServedMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, t.Context(), newServedMemoriesFixture(t, "hm09")) } -func TestServedMemoriesRecallConflatesStoredEmptyWithAbsent(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, t.Context(), newServedMemoriesFixture(t, "hm10")) +func TestServedMemoriesEmptyRowIsPresentUntilForgotten(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, t.Context(), newServedMemoriesFixture(t, "hm10")) } func TestServedMemoriesForgetRemovesExactlyTheNamedRow(t *testing.T) { diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index c367198b1..6cca206e6 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -558,10 +558,10 @@ func (c *Client) RememberMemory(ctx context.Context, body apigen.RememberRequest return &out, nil } -// RecallMemory reads one memory. A miss — including a row stored as the empty -// string — is a 404, which the problem mapper turns into issueops.ErrNotFound; -// the memory role converts that to its own Found-false result, because -// memoryops has deliberately no ErrNotFound. +// RecallMemory reads one memory. A miss is a 404, which the problem mapper +// turns into issueops.ErrNotFound; the memory role converts that to its own +// Found-false result, because memoryops has deliberately no ErrNotFound. A row +// stored as the empty string is not a miss: it is a 200 with an empty value. func (c *Client) RecallMemory(ctx context.Context, key string) (*apigen.Memory, error) { path, err := MemoryPath(key) if err != nil { diff --git a/internal/storage/dolt/memories.go b/internal/storage/dolt/memories.go index 8ec10f89f..c9da17c57 100644 --- a/internal/storage/dolt/memories.go +++ b/internal/storage/dolt/memories.go @@ -55,15 +55,18 @@ func (m *memories) Recall(ctx context.Context, req memoryops.RecallRequest) (mem if err != nil { return memoryops.RecallResult{}, err } - var value string + var ( + value string + found bool + ) if err := m.store.withReadTx(ctx, func(tx *sql.Tx) error { var err error - value, err = storagememoryops.RecallInTx(ctx, tx, key) + value, found, err = storagememoryops.RecallInTx(ctx, tx, key) return err }); err != nil { return memoryops.RecallResult{}, err } - return memoryops.RecallResult{Key: key, Value: value, Found: value != ""}, nil + return memoryops.RecallResult{Key: key, Value: value, Found: found}, nil } // Forget reads the value and deletes the row in ONE transaction, so the value diff --git a/internal/storage/dolt/memories_contract_test.go b/internal/storage/dolt/memories_contract_test.go index 4e7d0cdc4..0a304b59f 100644 --- a/internal/storage/dolt/memories_contract_test.go +++ b/internal/storage/dolt/memories_contract_test.go @@ -55,8 +55,8 @@ func TestMemoriesContract(t *testing.T) { t.Run("RecallReportsAMissAsNotFoundNotAnError", func(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, ctx, fixture) }) - t.Run("RecallConflatesStoredEmptyWithAbsent", func(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, ctx, fixture) + t.Run("EmptyRowIsPresentUntilForgotten", func(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, ctx, fixture) }) t.Run("ForgetRemovesExactlyTheNamedRow", func(t *testing.T) { conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, ctx, fixture) diff --git a/internal/storage/embeddeddolt/memories.go b/internal/storage/embeddeddolt/memories.go index 75d3f6ad4..d52b68a37 100644 --- a/internal/storage/embeddeddolt/memories.go +++ b/internal/storage/embeddeddolt/memories.go @@ -55,15 +55,18 @@ func (m *memories) Recall(ctx context.Context, req memoryops.RecallRequest) (mem if err != nil { return memoryops.RecallResult{}, err } - var value string + var ( + value string + found bool + ) if err := m.store.withConn(ctx, false, func(tx *sql.Tx) error { var err error - value, err = storagememoryops.RecallInTx(ctx, tx, key) + value, found, err = storagememoryops.RecallInTx(ctx, tx, key) return err }); err != nil { return memoryops.RecallResult{}, err } - return memoryops.RecallResult{Key: key, Value: value, Found: value != ""}, nil + return memoryops.RecallResult{Key: key, Value: value, Found: found}, nil } func (m *memories) Forget(ctx context.Context, req memoryops.ForgetRequest) (memoryops.ForgetResult, error) { diff --git a/internal/storage/embeddeddolt/memories_contract_test.go b/internal/storage/embeddeddolt/memories_contract_test.go index 28baf2b2c..cdcd9c266 100644 --- a/internal/storage/embeddeddolt/memories_contract_test.go +++ b/internal/storage/embeddeddolt/memories_contract_test.go @@ -54,8 +54,8 @@ func TestMemoriesContract(t *testing.T) { t.Run("RecallReportsAMissAsNotFoundNotAnError", func(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, ctx, fixture) }) - t.Run("RecallConflatesStoredEmptyWithAbsent", func(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, ctx, fixture) + t.Run("EmptyRowIsPresentUntilForgotten", func(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, ctx, fixture) }) t.Run("ForgetRemovesExactlyTheNamedRow", func(t *testing.T) { conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, ctx, fixture) diff --git a/internal/storage/memoryops/memories.go b/internal/storage/memoryops/memories.go index a9c58e1ff..31dce53e5 100644 --- a/internal/storage/memoryops/memories.go +++ b/internal/storage/memoryops/memories.go @@ -84,9 +84,7 @@ func MemoriesFromConfig(all map[string]string) map[string]string { // destroyed. // // replaced is about the ROW, not the value: a memory stored as the empty string -// reports true even though a Recall of it would report Found false. See -// memoryops.RememberResult.Replaced, which states that divergence rather than -// smoothing it over. +// reports true, just as Recall reports it found. func RememberInTx(ctx context.Context, tx *sql.Tx, key, content string) (replaced bool, err error) { storageKey := StorageKey(key) replaced, err = configRowExistsInTx(ctx, tx, storageKey) @@ -99,31 +97,33 @@ func RememberInTx(ctx context.Context, tx *sql.Tx, key, content string) (replace return replaced, nil } -// RecallInTx reads one memory in the caller's transaction. A key nothing stored -// answers "" with a nil error: the seam cannot tell that from a row stored -// empty, and the role's Found field is where that conflation is stated. -func RecallInTx(ctx context.Context, tx *sql.Tx, key string) (string, error) { - return issueops.GetConfigInTx(ctx, tx, StorageKey(key)) +// RecallInTx reads one memory in the caller's transaction and reports whether +// its row exists. A stored empty value is therefore distinct from no row. +func RecallInTx(ctx context.Context, tx *sql.Tx, key string) (value string, found bool, err error) { + storageKey := StorageKey(key) + err = tx.QueryRowContext(ctx, "SELECT value FROM config WHERE `key` = ?", storageKey).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("recall memory %s: %w", storageKey, err) + } + return value, true, nil } // ForgetInTx removes one memory and returns what it held, both in the caller's // transaction — which is what makes the returned value the one that was // actually deleted rather than the one an earlier read happened to see. // -// found is Recall's found, not the row's: a memory stored as the empty string -// reports found false and IS NOT DELETED. That keeps Forget's answer and -// Recall's answer the same statement, which is what a front door printing -// "No memory with key ..." after a successful Recall would otherwise contradict. -// // The delete names the one encoded key. There is no LIKE and no prefix sweep in // it, so forgetting "a" cannot take "a-b" with it. func ForgetInTx(ctx context.Context, tx *sql.Tx, key string) (previous string, found bool, err error) { storageKey := StorageKey(key) - previous, err = issueops.GetConfigInTx(ctx, tx, storageKey) + previous, found, err = RecallInTx(ctx, tx, key) if err != nil { return "", false, err } - if previous == "" { + if !found { return "", false, nil } if err := issueops.DeleteConfigInTx(ctx, tx, storageKey); err != nil { diff --git a/internal/storage/uow/memories.go b/internal/storage/uow/memories.go index fc5040881..3b8120468 100644 --- a/internal/storage/uow/memories.go +++ b/internal/storage/uow/memories.go @@ -88,11 +88,16 @@ func (m *memories) Recall(ctx context.Context, req memoryops.RecallRequest) (mem return memoryops.RecallResult{}, err } return RunTxRead(ctx, m.provider, func(ctx context.Context, uw UnitOfWork) (memoryops.RecallResult, error) { - value, err := uw.ConfigUseCase().GetConfig(ctx, storagememoryops.StorageKey(key)) + // The same whole-table probe as Remember, for the reason given there: + // Found is about the ROW, GetConfig answers "" for a missing row and an + // empty one alike, and the map's key set is the only place the + // difference survives. It is one query on a table of tens of rows. + all, err := uw.ConfigUseCase().GetAllConfig(ctx) if err != nil { return memoryops.RecallResult{}, err } - return memoryops.RecallResult{Key: key, Value: value, Found: value != ""}, nil + value, found := all[storagememoryops.StorageKey(key)] + return memoryops.RecallResult{Key: key, Value: value, Found: found}, nil }) } @@ -111,11 +116,14 @@ func (m *memories) Forget(ctx context.Context, req memoryops.ForgetRequest) (mem storageKey := storagememoryops.StorageKey(key) result, err := RunTxResult(ctx, m.provider, func(ctx context.Context, uw UnitOfWork) (memoryops.ForgetResult, string, error) { cfg := uw.ConfigUseCase() - previous, err := cfg.GetConfig(ctx, storageKey) + // The row probe Remember and Recall use: gating on GetConfig's "" is + // what left a memory stored empty unforgettable. + all, err := cfg.GetAllConfig(ctx) if err != nil { return memoryops.ForgetResult{}, "", err } - if previous == "" { + previous, found := all[storageKey] + if !found { return memoryops.ForgetResult{Key: key}, "", nil } if err := cfg.DeleteConfig(ctx, storageKey); err != nil { diff --git a/internal/storage/uow/memories_contract_test.go b/internal/storage/uow/memories_contract_test.go index 9725f306a..85865a199 100644 --- a/internal/storage/uow/memories_contract_test.go +++ b/internal/storage/uow/memories_contract_test.go @@ -56,8 +56,8 @@ func TestMemoriesContract(t *testing.T) { t.Run("RecallReportsAMissAsNotFoundNotAnError", func(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, ctx, fixture) }) - t.Run("RecallConflatesStoredEmptyWithAbsent", func(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, ctx, fixture) + t.Run("EmptyRowIsPresentUntilForgotten", func(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, ctx, fixture) }) t.Run("ForgetRemovesExactlyTheNamedRow", func(t *testing.T) { conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, ctx, fixture) diff --git a/memoryops/errors.go b/memoryops/errors.go index c63c53e15..126aed3c5 100644 --- a/memoryops/errors.go +++ b/memoryops/errors.go @@ -23,13 +23,8 @@ var ErrValidation = beadserrors.ErrValidation // THERE IS DELIBERATELY NO ErrNotFound ON THIS ROLE. // -// beadserrors declares one; this leaf does not re-export it. The storage seam -// beneath it cannot tell an absent config row from a row stored as the empty -// string (issueops/workspaceconfig.go states the same conflation for settings, -// and it is the same table). A role that answered a Recall of an unknown key -// with ErrNotFound would be minting an error out of a distinction it cannot -// actually see, and the first out-of-band empty write would make it a lie. -// -// Misses are RESULT-CARRIED instead — RecallResult.Found, ForgetResult.Found — -// and the front doors translate: the CLI to its SilentExit contract, an HTTP -// door to its 404. That keeps the invention where a door can justify it. +// beadserrors declares one; this leaf does not re-export it. Misses are +// RESULT-CARRIED instead — RecallResult.Found, ForgetResult.Found — and the +// front doors translate: the CLI to its SilentExit contract, an HTTP door to +// its 404. That lets callers handle absence without classifying an operational +// error and keeps presentation policy at the door. diff --git a/memoryops/memories.go b/memoryops/memories.go index fd36482de..7788f1761 100644 --- a/memoryops/memories.go +++ b/memoryops/memories.go @@ -40,11 +40,7 @@ type RememberResult struct { // observation and the act it describes. // // A previous value that was the EMPTY STRING reports Replaced true. The ROW - // existed, even though a Recall of that key would have answered Found - // false. That divergence is the storage seam's conflation showing through - // (see RecallResult.Found); it is stated here rather than smoothed over, - // because smoothing it would mean this field reporting "nothing was there" - // about a write that overwrote something. + // existed, regardless of what it held. Replaced bool } @@ -60,15 +56,9 @@ type RecallResult struct { Key string // Value is the stored content, or "" — and Found says how to read the "". Value string - // Found is Value != "". A MEMORY STORED AS THE EMPTY STRING AND AN ABSENT - // MEMORY ARE THE SAME ANSWER: Found false, nil error. - // - // No front door can create the first of those — Remember refuses empty - // content — but an out-of-band config write can, and this role does not - // invent a distinction the seam beneath it cannot see. List DOES enumerate - // such a row, because its KEY exists, and that is the one way a caller can - // tell the two apart: the same same-answer-different-row asymmetry the - // settings contract pins for `bd config get`. + // Found reports whether the row exists, independently of Value. A memory + // stored as the empty string therefore reports Found true; an absent memory + // reports Found false with the same empty Value. Found bool } @@ -84,9 +74,8 @@ type ForgetResult struct { // read in the SAME TRANSACTION as the delete, so it is what was actually // removed rather than what a previous read happened to see. Value string - // Found false means nothing was stored under Key (or an empty string was: - // the same conflation RecallResult.Found describes) and NOTHING WAS - // DELETED. + // Found false means no row was stored under Key and NOTHING WAS DELETED. A + // row holding the empty string is found and deleted. // // It is a result, not an error. The front doors own the not-found // presentation — the CLI has an exit-code contract for it — and a role that @@ -160,7 +149,7 @@ type Memories interface { // THE REFUSALS, in the order they are applied: // // - Content empty after trimming is ErrValidation. There is nothing to - // remember, and a row stored empty is a row Recall would then deny; + // remember; // - Content from which no key can be derived, when Key is empty, is // ErrValidation. The caller's recourse is an explicit key, which the // message says. From a2d43e6792d1951f29d85c36aaf4f24229550da6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 12 Oct 2026 19:50:34 +0000 Subject: [PATCH 09/99] fix(mol): molecule reads, auto-close and close --continue over http through issueops entries (S16) (#7513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(create): let a remote backend mint the --parent child id `bd create --parent` always called the raw GetNextChildID first, which the http backend does not serve, so every child create on a remote workspace refused before writing anything. The Lifecycle role already mints . for a create that carries a ParentID and no id of its own, on every backend and inside the create's own transaction; conformance RunIssueOperationsCreateUnderAParentMintsTheNextChildID pins it on every leg, the served one included (and the bd-enterprise postgres leg the MC server runs). So the create route needs no new capability token: on a registered Remote backend the client skips the raw reservation and lets the server mint. The local route keeps its reservation so its Dolt commit staging is unchanged, and an explicit --id beside --parent is still refused up front. Co-Authored-By: Claude Opus 5.5 * fix(mol): molecule reads, auto-close and close --continue over issueops roles Over the http backend three molecule paths failed, and two of them silently: - `bd mol current` / `bd mol progress` without an id answered "no molecules" because the in_progress listing (a raw SearchIssues shape) and the parent walk (raw GetDependencyRecordsForIssues) were refused and the refusal was swallowed; with an id they refused on raw GetMoleculeProgress. - The molecule auto-close after `bd close` never ran: findParentMolecules swallowed the same refused edge read into "not part of a molecule". - `bd close --continue` claimed the next step in a raw RunInTransaction. The direct route now reads molecules through roleMolStore, which answers through roles every backend serves: EdgeReader (batched at the wire's 100 anchors) for the parent walk, BatchGetter for the roots (Get per id where a server does not publish issues.batchGet), Relations for the step rows (full rows; the raw dependents read over http is getIssue's shallow projection, which zeroed step assignees) and the progress counts, and Reader.List for the in_progress/hooked listings. The auto-close closes the root through Lifecycle.Close, and the --continue claim is one Lifecycle.Update guarded by ExpectedStatus=open. No refusal is swallowed any more: findParentMolecules, the molecule finders and the hooked scan return their errors; `mol current/progress` report them; the auto-close stays best effort but warns on stderr; AdvanceToNextStep only moves on to the next ready step when the claim LOST a race (status mismatch, not claimable, already claimed) and returns anything else. The one heuristic still skipped is the id-pattern heal in loadDescendants, and only for an ErrUnsupported backend that cannot express an id-prefix search; other failures now surface. `mol current`/`mol progress` leave the types.IssueFilter forbidigo hole, since they no longer build a filter. gc does not depend on bd's molecule auto-close: it runs its own (cmd/gc/molecule_autoclose.go, from the bd on_close hook and the controller's bead-close events). Its worker prompts do run `bd mol current`. Served e2e (TestE2E_MoleculesOverHTTP): create --parent mints .1/.2, mol current is non-empty with the full in-progress step, mol progress counts, close --continue auto-advances, closing the last step auto-closes the root. `mol current` without an id now refuses out loud on this stack (asserted: never an empty list); its non-empty assertion is skipped until S6c (externaldeps reader passthrough) is underneath. Co-Authored-By: Claude Opus 5.5 * fix(mol): list molecule candidates with the four wire knobs, drop the S6c skip On top of S6c the in_progress listing behind 'bd mol current' (no id) reaches the v0 wire encoder, which refuses ListRequest.IncludeAllTypes. Ask for templates, gates, infra and ephemeral rows explicitly, which the wire publishes, and run the 'mol current without an id' served case. Co-Authored-By: Claude Opus 5.5 * feat(issueops): one library entry each for molecule auto-close, advance and reads The molecule rules lived only in cmd/bd, in copies that disagreed: two auto-close implementations that closed the root through different library entries outside the step close's transaction, a `close --continue` claim that set no assignee and could take a step another agent held on the direct and http routes, and three progress counts. Every molecule rule now has one implementation in the public issueops package, composed over three roles every backend serves (BatchGetter, Relations, EdgeReader): MoleculeRoots, LoadMolecule, AnalyzeMolecule, ViewMolecule, ReadMoleculeProgress, ActiveMoleculeIDs, CompletedMolecule. Membership is the parent-child edge. Auto-close: CloseRequest.AutoCloseMolecule and CloseBatchRequest.AutoCloseMolecule (opt-in, so gc's own auto-close is not doubled) close the molecule root a close completed IN THE SAME TRANSACTION, unforced and guarded on the root revision read there: storageissueops.CloseCompletedMoleculeInTx on the store backends (the role bodies bound to the caller's tx, TxMoleculeReader) and uow.CloseCompletedMoleculeInUOW on the unit of work. It replays on an idempotent re-close, reports the root on CloseResult/CloseOutcome .AutoClosedMolecule, and a root its own close policy refuses stays open and is reported (MoleculeAutoCloseRefusal) without failing the step. Advance: a new MoleculeStepper role (storage.DoltStorage accessor, uow.MoleculeStepperSource) whose one implementation, issueops.NewMoleculeStepper, claims the next ready step through the Claimer (Lifecycle.Update's Claim for a wisp step): assignee set, a step another actor holds skipped, never taken. HTTP: closeIssue/batchCloseIssues carry auto_close_molecule behind the issues.close.autoCloseMolecule behavior token, and POST /v0/beads/issues/{id}:advanceMolecule (issues.advanceMolecule) serves the stepper. Handlers only decode, call the role and encode. The client sends the member and dials the operation only where advertised; against an older server it DEGRADES rather than refuses — it runs the same library rule over its own roles (issueops.CloseCompletedMolecule after the close, NewMoleculeStepper for the advance), so the outcome is the same in more round trips. The capability fallbacks that lived in cmd/bd move into the client: BatchGetter answers one getIssue per id without issues.batchGet, and EdgeReader splits past the operation's 100 anchors. Conformance: RunMolecule* on dolt, embedded, uow and http (the http leg twice: served, and against a server with the tokens struck out). Co-Authored-By: Claude Opus 5.5 * refactor(cmd/bd): molecule commands and create --parent through the library entries `bd close`, `bd close --continue`, `bd mol current`, `bd mol progress`, `bd ready --mol` and `bd create --parent` now call the one library entry for each rule on every route (direct, proxied server, remote): - close (batch, --if-revision, proxied) sets CloseRequest/ CloseBatchRequest.AutoCloseMolecule and only reports what the library closed in the step close's transaction. autoCloseCompletedMolecule, autoCloseProxiedCompletedMolecule and closeCompletedMoleculeRoot are gone, and so is the separate post-close write. - --continue is the route's MoleculeStepper role: a real claim (the assignee is set) that skips a step another actor holds. Before, the direct and http routes flipped the status only and could take it. AdvanceToNextStep, ClaimStepIfOpen and its two implementations are gone. - mol current / mol progress / ready --mol read through issueops.ViewMolecule, ReadMoleculeProgress, ActiveMoleculeIDs and LoadMolecule/AnalyzeMolecule over the route's roles. Over http this fixes the missing step assignee on `ready --mol` (corpus mark mc-ready-mol). The proxied route's SearchIssues fallback, the role store, the third progress count and the cmd/bd copy of the readiness analysis are deleted; only presentation stays here. - create --parent asks the create role to mint the child id on every route (the local CLI reservation is gone) and to inherit the parent's labels (InheritLabelsFromParent) inside the create. The dry-run preview still shows the labels, and a failed label read there is reported, not swallowed. TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute runs one molecule lifecycle through the real binary on the direct, proxied and remote (`bd connect` to `bd serve`) routes and requires the same rows and the same JSON on all three. The served e2e now also pins the held-step skip, the claimed step's assignee and the full rows of `ready --mol`. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): refuse molecule auto-close and advance before dialing an older server The client used to DEGRADE against a server without issues.close.autoCloseMolecule / issues.advanceMolecule: it closed the step without the member and ran a second, client-side copy of the auto-close (issueops.CloseCompletedMolecule) in its own transactions, and composed the stepper over its own roles. That kept two executions of each rule alive. Now the rule runs in ONE place, the server's library: a close or batch close that sets AutoCloseMolecule, and an Advance, refuse before anything is dialed with the typed capability error (storage.ErrUnsupported, Capability naming the token). The composed single-item batch leg now carries the member too. - issueops.CloseCompletedMolecule (the role-composed form) is gone. MoleculeRootClose is the one root close request (unforced, the auto-close reason, the session, guarded on the root revision read), and both in-transaction bodies (storage issueops CloseCompletedMoleculeInTx, uow CloseCompletedMoleculeInUOW) build from it. - MoleculeView.Ready is `bd ready --mol`'s list (every analysis-ready node, hydrated), so ready --mol and mol current derive from ViewMolecule alone. Tests: unit pre-dial refusal and member pins (httpclient molecule_test.go), TestServedMoleculeRefusesAgainstAnOlderServer (step stays open, nothing claimed) replacing the degraded contract arm. Co-Authored-By: Claude Opus 5.5 * fix(cmd/bd): ask for the molecule auto-close only where the route serves it The refuse-before-dialing client made every `bd close` over http fail against a server that does not advertise issues.close.autoCloseMolecule (an older bd serve, or the MC server shape, whose orchestrator owns the molecule lifecycle): the CLI always set AutoCloseMolecule. storage.ServesMoleculeAutoClose answers, through every decorator, whether a store's closes may carry the flag (MoleculeAutoCloseProber, implemented by httpclient.Store from its handshake; every local backend serves it). The direct batch close and the --if-revision close set the flag from it; the proxied route always serves it. Against a silent server the close goes out without the member and the server's side owns the auto-close. Co-Authored-By: Claude Opus 5.5 * test(backend/http): scope the advertised-policy check to reads before the close The close policy check is the client-side policy's last chance: it runs before the close request, so only the reads up to that request count. A read after the close (the old-server molecule notice reads the closed step's edges) is not the external-dependency policy. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): drop the six marks S16 and S19 retire With the molecule library entries underneath, mol current (with and without an id, json and text), mol progress and ready --mol read through issueops.LoadMolecule over the roles, and create --parent lets the server mint the child id. Those six shapes pass on the reference profile. Co-Authored-By: Claude Opus 5.5 * fix(cmd/bd): never silent against a server without the molecule tokens Against a server that advertises neither issues.close.autoCloseMolecule nor issues.advanceMolecule (an older bd serve, or one whose orchestrator owns the molecule lifecycle): - `bd close --continue` closed the step and only then found the advance refused, leaving the step closed and the next step unclaimed. It now asks storage.ServesMoleculeAdvance (a MoleculeAdvanceProber the http store answers from its handshake; local backends always serve it) BEFORE the close and refuses up front, saying nothing was closed. - `bd close` went out without auto_close_molecule and said nothing. When the closed step belongs to a molecule (issueops.MoleculeRoot), a notice on stderr now names the molecule and the missing token: its auto-close is the server's to run. The batch and --if-revision closes both say it. The served e2e masks both tokens out of the handshake and checks the up-front refusal (the step stays open) and the notice; the httpclient unit test pins ServesMoleculeAdvance to the handshake. Co-Authored-By: Claude Opus 5.5 * refactor(cmd/bd): mol show through issueops.LoadMolecule on every route `bd mol show` still loaded the molecule through cmd/bd's loadTemplateSubgraph (raw GetIssue, GetDependentsWithMetadata and GetDependencyRecords, plus an id-pattern child probe), while mol current, mol progress and ready --mol read through the library. It now loads through issueops.LoadMolecule over the route's molecule roles (the direct store's, or the proxied provider's), so membership is the parent-child edge everywhere and the http route reads through roles the server serves. The served e2e checks mol show's root and step count. Co-Authored-By: Claude Opus 5.5 * docs: the molecule auto-close fails the step's close on a non-policy error; stale comments CloseRequest.AutoCloseMolecule and CloseCompletedMoleculeInTx now say what changed when the auto-close moved into the close's transaction: a root close the close policy refuses still leaves the root open and is reported, but any other failure (a storage error, a lost race on the root's revision) fails the step's close with it, where the CLI used to close the root afterwards and only warn. Stale: the advanceMolecule vocabulary entry said the client composes the stepper over its own roles when unserved (it refuses before dialing), and the graph-create collision probe called itself best-effort by transport (the http GetIssue miss is storage.ErrNotFound like every other store's). Co-Authored-By: Claude Opus 5.5 * docs(changelog): the molecule tokens and the S16/S19 behaviours Entries for issues.close.autoCloseMolecule and issues.advanceMolecule, the issueops molecule entries, the auto-close inside the step's close (and a non-policy failure of it failing the close), the real claim behind close --continue, the molecule reads on every route, the notice and the up-front --continue refusal against a server without the tokens, and create --parent / mol reads over http. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): regenerate unsupported_gen.go for MoleculeStepper The PR adds MoleculeStepper() to storage.DoltStorage (200 -> 201 methods). httpclient implements it by hand, but the generated shell's trailer still counted 200, so //internal/storage/unsupportedgen:unsupportedgen_test (TestGeneratedShellIsFresh) failed in Bazel / test and Bazel / integration, turning CI Gate / Required red. Regenerated with `go generate ./internal/httpclient`; the only change is the trailer count (143 of 201 methods generated; 58 left). Co-Authored-By: Claude Opus 5.5 * fix(molecule): auto-close refusals, write skew, close --json, docs Review fixes for #7513, numbered as in each iteration's review synthesis. Iteration 1: F1: a close guard's refusal of the root (TemplateReadOnlyError, PinnedError, CloseNotAssigneeError) is an auto-close refusal, like the close policy's: the root stays open and is reported, and the step's close stands. Pinned by the new conformance case RunMoleculeGuardedRootKeepsTheStepClose on dolt, embedded, uow and the served http client. F2: two closes that finish a molecule's last two steps in concurrent transactions each read the other's step as open, so neither closed the root and both committed. Every auto-close now rewrites the root's coordination cell before it decides the root (local_metadata molecule-coordination/v1/, the dependency coordination idiom and sharding with a FreshRowLock value), in both twins (CloseCompletedMoleculesInTx and CloseCompletedMoleculesInUOW). Two deciders of one root now conflict at commit (1213), and the retry reads the winner's close and closes the root. This is the synthesis's option (c), applied through a coordination cell instead of the root row; it needs no post-commit second transaction and no engine lock. The deterministic two-transaction regression fails without the touch and passes with it on dolt and uow; the concurrent-close tests end with the root closed in every round, reported by exactly one close. F3: the auto-close checks the root's eligibility before walking any step, MoleculeStepsClosed stops at the first open step, and a batch decides each root once and reports it on the first item under it. F4: ErrVersionMismatch on the root is not a refusal; it fails the step's close with it. The classifier unit test pins this. A conformance case cannot stage that race deterministically through the public roles. F5: storage.ServesMoleculeAdvance answers (bool, error), and bd close --continue refuses before closing anything when the handshake cannot be read. F6: the CHANGELOG and the stepper and batch-closer doc comments say what the code does. F7: the unserved-auto-close notice fires only when the close left an auto-closing molecule with every step closed and its root open (issueops.CompletedMolecule): again on a re-close, once per root in a batch, never for a step under an ordinary epic or an unfinished molecule. Proxied mol show, mol current and mol progress again report a read unit of work that cannot be opened as itself, not as a molecule that was not found. F8: the write-ledger comment says the close is refused before dialing. The dead FirstClosed/LastClosed guards in mol progress pre-exist on main and are left as they are. Iteration 2: F1: on the proxied route, bd ready --mol reported a read unit of work it could not open twice: once as itself, then as "molecule '' not found", which under --json is a second error document on stdout. It now returns moleculeNotResolved, as mol show, mol current and mol progress do. TestProxiedReadyMolReportsAnUnopenableReadOnce drives runMoleculeReady with a provider whose NewUOW fails and wants one Error: line, or one JSON document; both subtests fail with the old return. F2: the coordination cell serializes every concurrent close of a step under an open auto-closing root, not only the closes that finish the molecule. That is the price of deciding the root in the step's transaction, so it is documented (TouchMoleculeCoordinationInTx, the UOW twin, CloseRequest.AutoCloseMolecule, the CHANGELOG) and pinned on dolt and uow: two sibling closes with a third step still open conflict (both commit when the touch is removed, on either twin), and a six-step fan-out closes every step through the store's retry, with the root closed and reported by exactly one close. The synthesis's alternative, a post-commit recheck by any close that read a step open, is not taken: it is a second, best-effort transaction after the step's commit, the shape this PR exists to remove. F3: MoleculeRootClose's doc no longer promises that a step reopened between the completion read and the root close fails the step's close. The guard is the root's RowVersion, which a step reopen never writes; the doc now says what the guard covers and that this race goes undetected (it is inherited from the CLI's old post-commit auto-close). F4: the user docs describe the auto-close. docs/workflows/molecules.md replaces "Closing the last child does not close the molecule root" with which roots close and which stay open, the refusal warning, the unserved-server note and the --json members, and documents the claiming --continue and parent-child step membership; docs/reference/json-schema.md gains a bd close --json section. These are main-only behaviors on pages pinned to v1.2.2 (docs/cli-docs.pin): a maintainer call against §1 of engdocs/decisions/2026-07-17-docs-release-pin.md, as in #6508, #5855, #5997 and #5092. The auto-close itself shipped in v1.2.2 (cmd/bd/close.go autoCloseCompletedMolecule), so the old sentence was already wrong at the pin. docs/cli-reference/ is generated from the pinned release and is not hand-edited; the bd close and bd mol show help texts carry the same facts and reach it at the next pin bump. F5: bd close --json reports the auto-close. The entry of a close that completed a molecule gains auto_closed_molecule (the root it closed) or molecule_auto_close_refusal (why the root stayed open), the HTTP API's names, on the direct, proxied and --if-revision routes. The members sit beside the issue's own rather than wrapping the output in an object the way --suggest-next does: an object would change the output's shape exactly when a close completes a molecule and break every consumer that reads an array, while members leave each entry decoding as the closed issue, compose with `closed` under the flag objects, and leave a plain close's bytes unchanged. Pinned on the embedded and proxied tiers (TestEmbeddedCloseJSONReportsMoleculeAutoClose, TestProxiedServerClose2B json_reports_*, and both --if-revision replay tests), all red without the change. F6: cmd/bd's BeadsTemplateLabel is issueops.MoleculeTemplateLabel, so the template commands and molecule membership cannot drift apart. F7: the UOW twin's revision probe says that it writes nothing, and the BatchCloseRequest.auto_close_molecule spec description has its verb back (apigen regenerated, no drift). Iteration 3: F1: on the unit-of-work route (bd serve, the proxied CLI), a refused root's molecule_auto_close_refusal and warning carried the use case's and repository's wrapping ("close : db: IssueSQLRepository.CloseChecked : ...") around the refusal, where the tx twin and the docs give the refusal alone. Both twins now report it through RefusedMoleculeAutoClose, which answers the refusing guard's or policy's own sentence. RunMoleculeRefusedRootKeepsTheStepClose, RunMoleculeGuardedRootKeepsTheStepClose and the embedded and proxied close --json tests want that exact sentence; the uow conformance leg and the proxied test fail without the unwrap. F2: the docs no longer promise ErrVersionMismatch for a write to the root that races its auto-close. The root's read and its revision check share one transaction snapshot, so that write conflicts with the root close at commit and the retry decides the root again; the close fails only if it still conflicts when its retry budget runs out (MoleculeRootClose, IsMoleculeAutoCloseRefusal, CloseRequest.AutoCloseMolecule, the tx twin's comment, the CHANGELOG and the OpenAPI description, apigen regenerated). F3: TestEmbeddedCloseJSONReportsMoleculeAutoClose moves above TestEmbeddedCloseConcurrent's doc comment, which documents its own test again. F4: bd mol progress counting direct steps while the auto-close counts every descendant predates this PR; it is left for a follow-up. F5: cmd/bd's molReader drops GetMoleculeProgress, which nothing called once mol progress read through issueops.ReadMoleculeProgress. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .../scripts/embedded-storage-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + .golangci.yml | 8 +- CHANGELOG.md | 74 +- backend/conformance/BUILD.bazel | 1 + backend/conformance/molecule_contract.go | 480 ++++++++++ backend/http/BUILD.bazel | 1 + backend/http/corpus/testdata/corpus.yaml | 6 - backend/http/e2e_mol_test.go | 305 ++++++ backend/http/e2e_remote_roles_test.go | 13 +- cmd/bd/BUILD.bazel | 2 + cmd/bd/close.go | 192 ++-- cmd/bd/close_direct.go | 10 +- cmd/bd/close_embedded_test.go | 113 +++ cmd/bd/close_if_revision.go | 68 +- cmd/bd/close_if_revision_embedded_test.go | 6 +- cmd/bd/close_if_revision_proxied_test.go | 6 +- cmd/bd/close_proxied_integration_test.go | 26 + cmd/bd/close_proxied_server.go | 158 ++-- cmd/bd/create.go | 58 +- cmd/bd/critical_paths_test.go | 29 +- cmd/bd/graph_apply.go | 6 +- cmd/bd/mol_bond_compound_label_test.go | 3 - cmd/bd/mol_current.go | 566 ++---------- cmd/bd/mol_port.go | 69 -- cmd/bd/mol_progress.go | 150 ++- cmd/bd/mol_proxied_server.go | 163 +--- cmd/bd/mol_ready_gated.go | 11 +- cmd/bd/mol_roles.go | 233 +++++ cmd/bd/mol_route_parity_integration_test.go | 140 +++ cmd/bd/mol_show.go | 312 +------ cmd/bd/mol_show_proxied_integration_test.go | 114 +++ cmd/bd/mol_test.go | 154 ++-- cmd/bd/ready.go | 51 +- cmd/bd/ready_proxied_server.go | 91 +- cmd/bd/template.go | 21 +- docs/reference/json-schema.md | 23 + docs/workflows/molecules.md | 57 +- internal/httpapi/BUILD.bazel | 2 + internal/httpapi/advance_molecule.go | 64 ++ internal/httpapi/advance_molecule_test.go | 238 +++++ internal/httpapi/apigen/types.gen.go | 56 +- internal/httpapi/batch_close.go | 17 +- internal/httpapi/claim.go | 7 + internal/httpapi/close.go | 41 +- internal/httpapi/problem.go | 10 + internal/httpapi/routes.go | 21 + internal/httpapi/server.go | 20 + internal/httpapi/server_test.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 172 +++- .../httpapi/wireshape/testdata/golden.json | 87 ++ internal/httpclient/BUILD.bazel | 3 + internal/httpclient/batchcloser.go | 30 +- internal/httpclient/batchgetter.go | 46 + internal/httpclient/lifecycle.go | 24 +- internal/httpclient/molecule.go | 125 +++ internal/httpclient/molecule_test.go | 180 ++++ internal/httpclient/roles_graph.go | 28 + internal/httpclient/served_molecule_test.go | 235 +++++ internal/httpclient/unsupported_gen.go | 2 +- internal/httpclient/wire/handshake.go | 14 + internal/httpclient/wire/paths.go | 14 +- internal/httpclient/wire/skew_test.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/write_ledger_test.go | 16 +- internal/storage/BUILD.bazel | 1 + internal/storage/dolt/BUILD.bazel | 2 + .../storage/dolt/molecule_contract_test.go | 266 ++++++ internal/storage/dolt/molecule_stepper.go | 17 + internal/storage/embeddeddolt/BUILD.bazel | 2 + .../embeddeddolt/molecule_contract_test.go | 63 ++ .../storage/embeddeddolt/molecule_stepper.go | 19 + .../storage/externaldeps/uow_decorator.go | 7 + internal/storage/hook_batch_closer.go | 3 + internal/storage/hook_issue_operations.go | 5 + internal/storage/hook_molecule_stepper.go | 31 + internal/storage/issueops/close_batch.go | 35 +- internal/storage/issueops/close_guard_test.go | 22 + internal/storage/issueops/execution.go | 11 +- internal/storage/issueops/molecule.go | 191 ++++ internal/storage/storage.go | 66 ++ internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/batch_closer.go | 18 + internal/storage/uow/batchgetter.go | 7 +- internal/storage/uow/edgereader.go | 73 +- internal/storage/uow/issue_operations.go | 11 +- internal/storage/uow/molecule.go | 182 ++++ .../storage/uow/molecule_contract_test.go | 257 ++++++ internal/storage/uow/notifying.go | 5 + internal/storage/uow/relations.go | 48 +- internal/telemetry/BUILD.bazel | 1 + internal/telemetry/molecule_stepper.go | 35 + issueops/BUILD.bazel | 8 +- issueops/batchcloser.go | 13 + issueops/issueops.go | 50 + issueops/molecule.go | 869 ++++++++++++++++++ issueops/molecule_active.go | 141 +++ issueops/molecule_internal_test.go | 40 + issueops/molecule_test.go | 595 ++++++++++++ issueops/moleculestepper.go | 209 +++++ 101 files changed, 6881 insertions(+), 1617 deletions(-) create mode 100644 backend/conformance/molecule_contract.go create mode 100644 backend/http/e2e_mol_test.go create mode 100644 cmd/bd/mol_roles.go create mode 100644 cmd/bd/mol_route_parity_integration_test.go create mode 100644 internal/httpapi/advance_molecule.go create mode 100644 internal/httpapi/advance_molecule_test.go create mode 100644 internal/httpclient/molecule.go create mode 100644 internal/httpclient/molecule_test.go create mode 100644 internal/httpclient/served_molecule_test.go create mode 100644 internal/storage/dolt/molecule_contract_test.go create mode 100644 internal/storage/dolt/molecule_stepper.go create mode 100644 internal/storage/embeddeddolt/molecule_contract_test.go create mode 100644 internal/storage/embeddeddolt/molecule_stepper.go create mode 100644 internal/storage/hook_molecule_stepper.go create mode 100644 internal/storage/uow/molecule.go create mode 100644 internal/storage/uow/molecule_contract_test.go create mode 100644 internal/telemetry/molecule_stepper.go create mode 100644 issueops/molecule.go create mode 100644 issueops/molecule_active.go create mode 100644 issueops/molecule_internal_test.go create mode 100644 issueops/molecule_test.go create mode 100644 issueops/moleculestepper.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index d8ad41995..0ac8f3440 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -214,6 +214,7 @@ 100 70 TestEmbeddedPurgeConcurrent 100 70 TestEmbeddedRoutedMutatingSiblingAssignTagDepTag +100 71 TestEmbeddedCloseJSONReportsMoleculeAutoClose 100 71 TestEmbeddedInitRoleRoutingExplicitDefault 100 71 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 8e103b3e0..714ee1098 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -282,6 +282,7 @@ 40 15 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary 40 15 TestGetIssue 40 15 TestLegacyTrackedIgnoredCursorHealsOnOpen +40 15 TestMoleculeContract 40 15 TestReadyWorkPageTotalFilterMatrix 40 16 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 938cf3ed6..a752c2b07 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -78,6 +78,7 @@ 34 7 TestProxiedServerIfRevisionGuardDelete 34 7 TestProxiedServerIfRevisionTargetGoneIsPreconditionFailed 34 7 TestProxiedServerLabelC +34 7 TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute 34 7 TestProxiedServerTypesAgreeWithCreate 34 8 TestProxiedServerClose2 diff --git a/.golangci.yml b/.golangci.yml index 5af1120fe..238e8fabf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -472,7 +472,13 @@ linters: linters: - forbidigo # Molecule reporting: each selects the members of one molecule. - - path: '^cmd/bd/mol_(current|port|progress|ready_gated|stale)\.go$' + # + # `mol current` and `mol progress` LEFT this list when their in_progress + # and hooked listings moved onto issueops.Reader.List (cmd/bd/mol_roles.go) + # so a remote backend answers them. The proxied route's in-transaction + # port still searches through the unit of work, and that filter lives in + # mol_port.go beside the port. + - path: '^cmd/bd/mol_(port|ready_gated|stale)\.go$' text: 'do not write a filter here' linters: - forbidigo diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ab3cbdec..bc6841359 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -206,6 +206,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 malformed `BEADS_MAX_ROWS`, which is ignored with a warning, a malformed cap value fails every unlimited ready or query read rather than falling back to the default. +- Two molecule capability tokens on `bd serve`. `issues.close.autoCloseMolecule` + announces that `POST /v0/beads/issues/{id}:close` and `POST + /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer + `auto_closed_molecule` and `molecule_auto_close_refusal`; + `issues.advanceMolecule` is the new `POST + /v0/beads/issues/{id}:advanceMolecule`, `bd close --continue`'s advance. + Both run the one library entry (`issueops.CloseRequest.AutoCloseMolecule`, + `issueops.MoleculeStepper`) on the server. The public `issueops` package + gains the molecule rules as one implementation over the BatchGetter, + Relations and EdgeReader roles (`MoleculeRoots`, `LoadMolecule`, + `AnalyzeMolecule`, `ViewMolecule`, `ReadMoleculeProgress`, + `ActiveMoleculeIDs`, `CompletedMolecule`, `MoleculeRootClose`); a + molecule's membership is the parent-child edge. Against a server that + does not advertise a token the HTTP client refuses a library close that + sets `AutoCloseMolecule`, or an advance, before dialing, with a typed + capability error; it never runs a client-side copy of either rule. +- `bd close --json` reports the molecule auto-close. The entry of a close + that completed a molecule carries the root it closed as + `auto_closed_molecule`, or why the root stayed open as + `molecule_auto_close_refusal`, on every route; before, that refusal + reached only stderr. Both members are absent from every other entry, so + each entry still decodes as the closed issue, and the output stays an + array (under `closed` with `--suggest-next`, `--continue` or + `--claim-next`). ### Changed @@ -243,6 +267,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 migrate-personal` and the missing-id lookup fallback all read the same local-only answer, and none pays a settings round trip for it. Local backends still fall back to the database values. +- **The molecule auto-close runs inside the step's close, and an error in + it fails the close.** `bd close` of a molecule's last step closes the + root in the same transaction (opt-in `AutoCloseMolecule`, so an + orchestrator that runs its own molecule lifecycle is not doubled), and it + replays on an idempotent re-close. The root close is unforced, so a root + a direct close would refuse stays open and is reported while the step's + close stands: one its close policy refuses (such as a live blocker), and + now also one a close guard refuses (a template, a pinned root, a root + another actor holds). Any other failure of the root close (such as a + storage error) now fails the step's close with it, where the CLI used to + close the root afterwards and only warn. + Two closes that finish a molecule's last steps at the same time no + longer leave its root open: one of them retries and closes it. The cost + is that concurrent closes of steps under one open auto-closing root now + serialize, whether or not either finishes the molecule: each conflict is + retried, and a close still conflicting when its retry budget runs out + fails with the serialization error. A batch close reports each root once, + on the first item under it that closed. +- **`bd close --continue` claims the next step for real.** The advance is + the `MoleculeStepper` role on every route: the assignee is set and a step + another agent holds is skipped, never taken (the direct and http routes + used to flip the status only). +- **`bd mol current`, `bd mol progress`, `bd ready --mol` and `bd mol show` + read through the library's molecule entries on every route**, so + membership is the parent-child edge everywhere (`bd mol show` no longer + adds children found only by an id pattern). +- **Against a server without the molecule tokens, `bd close` says what it + did not do.** If the server does not advertise + `issues.close.autoCloseMolecule` (an older `bd serve`, or one whose + orchestrator owns the molecule lifecycle), the close goes out without the + auto-close request. When that leaves a molecule with every step closed and + its root open, a notice on stderr names the root, once per root in a batch + and again on a re-close; a step under an ordinary epic or an unfinished + molecule prints nothing. If it does not advertise + `issues.advanceMolecule`, or the handshake that would say so fails, `bd + close --continue` refuses before closing anything, so the step is never + left closed with the next step unclaimed. - Every `bd` invocation used to start two git subprocesses before doing any work: `git rev-parse` to locate the repository and `git config user.name` to resolve the actor. On Linux the repository is now located in-process @@ -333,9 +394,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 rebuilds the same typed errors. `bd close` prints the same lines and exits as before, and `bd close --if-revision`, which skipped the guards, now applies them too. Two paths stay outside the guards. The raw storage - `CloseIssue`, which the molecule auto-close, a `close` line of `bd batch` - and the other closes `cmd/bd` makes outside `bd close` use, runs neither the - guards nor close policy on dolt, embedded or proxied; over HTTP it now sends + `CloseIssue`, which a `close` line of `bd batch` and the other closes + `cmd/bd` makes outside `bd close` use, runs neither the guards nor close + policy on dolt, embedded or proxied; over HTTP it now sends `force`, so there it also stops applying close policy, and only a template refuses. A status update into the done category (`bd update -s closed`, a `PATCH` whose `status` is done, an `update` item of `issues:batchApply`) @@ -457,6 +518,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 negative duration and fails, even though `--age 0s`'s own contract is that every wisp qualifies. `ageThreshold <= 0` now bypasses the clock comparison entirely (#7236). +- `bd create --parent` works on the HTTP backend: the create role mints the + `.` child id (and inherits the parent's labels) inside the + create on every route, instead of the CLI reserving it through a raw + `GetNextChildID` the HTTP backend does not serve. `bd mol current` (with + or without an id) and `bd mol progress` no longer answer empty or refuse + over http, and `bd ready --mol` step rows carry their assignee and + timestamps. - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index f5c8882cc..128d7ca97 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -41,6 +41,7 @@ go_library( "lifecycle_update_contract.go", "memories_contract.go", "metadata_cas_contract.go", + "molecule_contract.go", "portable.go", "querier_contract.go", "reader_contract.go", diff --git a/backend/conformance/molecule_contract.go b/backend/conformance/molecule_contract.go new file mode 100644 index 000000000..36021d53a --- /dev/null +++ b/backend/conformance/molecule_contract.go @@ -0,0 +1,480 @@ +package conformance + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the contract for the molecule operations every backend +// serves through one library entry each: +// +// - CloseRequest.AutoCloseMolecule / CloseBatchRequest.AutoCloseMolecule — +// the close closes the molecule root it completed IN ITS OWN TRANSACTION +// (internal/storage/issueops.CloseCompletedMoleculeInTx on the store +// backends, uow.CloseCompletedMoleculeInUOW on the unit of work, the +// server's own on http). Opt-in; unforced; guarded on the root revision. +// - issueops.MoleculeStepper — the advance behind `bd close --continue`, +// whose claim is a real claim (assignee = Actor) that never takes a step +// another actor holds. +// +// THE LEAF DOC COMMENTS ARE THE SPEC (issueops/issueops.go CloseRequest, +// issueops/batchcloser.go, issueops/moleculestepper.go, issueops/molecule.go). + +// MoleculeFixture supplies adapter-specific storage access for the molecule +// assertions. +type MoleculeFixture struct { + // IssuePrefix namespaces the ids each assertion seeds. + IssuePrefix string + Lifecycle publicops.Lifecycle + BatchCloser publicops.BatchCloser + Stepper publicops.MoleculeStepper + // CreateIssue and CreateWisp seed one row in the issues and wisps plane. + CreateIssue func(context.Context, *types.Issue, string) error + CreateWisp func(context.Context, *types.Issue, string) error + // AddDependency seeds one edge. + AddDependency func(context.Context, *types.Dependency, string) error + // QueryScalar runs a single-row query and scans it. + QueryScalar func(context.Context, string, []any, ...any) error +} + +const moleculeActor = "molecule-contract" + +type moleculeSeed struct { + t *testing.T + ctx context.Context + fixture MoleculeFixture +} + +func (s moleculeSeed) issue(id string, typ types.IssueType, status types.Status, ephemeral bool, labels ...string) { + s.t.Helper() + issue := &types.Issue{ID: id, Title: "molecule " + id, IssueType: typ, Status: status, Priority: 2, Labels: labels, Ephemeral: ephemeral} + if status == types.StatusClosed { + issue.CloseReason = "seeded closed" + } + create := s.fixture.CreateIssue + if ephemeral { + create = s.fixture.CreateWisp + } + if err := create(s.ctx, issue, moleculeActor); err != nil { + s.t.Fatalf("seed %s: %v", id, err) + } +} + +func (s moleculeSeed) edge(from, to string, typ types.DependencyType) { + s.t.Helper() + if err := s.fixture.AddDependency(s.ctx, &types.Dependency{IssueID: from, DependsOnID: to, Type: typ}, moleculeActor); err != nil { + s.t.Fatalf("seed edge %s -> %s: %v", from, to, err) + } +} + +// row reads id's status and assignee from whichever plane holds it. +func (s moleculeSeed) row(id string) (status, assignee string) { + s.t.Helper() + err := s.fixture.QueryScalar(s.ctx, + `SELECT status, COALESCE(assignee, '') FROM issues WHERE id = ? + UNION ALL SELECT status, COALESCE(assignee, '') FROM wisps WHERE id = ?`, + []any{id, id}, &status, &assignee) + if err != nil { + s.t.Fatalf("read %s: %v", id, err) + } + return status, assignee +} + +// molecule seeds root <- step1 (closed), step2 (open) and answers the ids. +func (s moleculeSeed) molecule(name string, rootType types.IssueType, ephemeral bool, labels ...string) (root, done, last string) { + s.t.Helper() + root = s.fixture.IssuePrefix + "-" + name + done, last = root+".1", root+".2" + s.issue(root, rootType, types.StatusOpen, ephemeral, labels...) + s.issue(done, types.TypeTask, types.StatusClosed, ephemeral) + s.issue(last, types.TypeTask, types.StatusOpen, ephemeral) + s.edge(done, root, types.DepParentChild) + s.edge(last, root, types.DepParentChild) + return root, done, last +} + +func (s moleculeSeed) closeStep(id string, autoClose bool) publicops.CloseResult { + s.t.Helper() + res, err := s.fixture.Lifecycle.Close(s.ctx, publicops.CloseRequest{ + Actor: moleculeActor, IssueID: id, Reason: "step done", Session: "molecule-session", AutoCloseMolecule: autoClose, + }) + if err != nil { + s.t.Fatalf("close %s: %v", id, err) + } + return res +} + +// RunMoleculeCloseAutoClosesTheCompletedRoot pins CloseRequest.AutoCloseMolecule: +// closing the last open step closes an auto-closing root in the same call, +// reports it, and records the auto-close reason. +func RunMoleculeCloseAutoClosesTheCompletedRoot(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + for _, shape := range []struct { + name string + rootType types.IssueType + labels []string + }{ + {"poured", types.TypeMolecule, nil}, + {"template", types.TypeEpic, []string{publicops.MoleculeTemplateLabel}}, + } { + root, _, last := s.molecule("auto-"+shape.name, shape.rootType, false, shape.labels...) + res := s.closeStep(last, true) + if !res.Changed || res.Issue == nil || res.Issue.Status != types.StatusClosed { + t.Fatalf("%s: close %s = %+v, want the step closed", shape.name, last, res) + } + if res.AutoClosedMolecule == nil || res.AutoClosedMolecule.ID != root { + t.Fatalf("%s: AutoClosedMolecule = %+v, want %s", shape.name, res.AutoClosedMolecule, root) + } + if res.MoleculeAutoCloseRefusal != "" { + t.Errorf("%s: MoleculeAutoCloseRefusal = %q, want none", shape.name, res.MoleculeAutoCloseRefusal) + } + if status, _ := s.row(root); status != string(types.StatusClosed) { + t.Errorf("%s: root %s status = %s, want closed", shape.name, root, status) + } + var reason string + if err := fixture.QueryScalar(ctx, `SELECT close_reason FROM issues WHERE id = ?`, []any{root}, &reason); err != nil { + t.Fatalf("read %s close_reason: %v", root, err) + } + if reason != publicops.MoleculeAutoCloseReason { + t.Errorf("%s: root close_reason = %q, want %q", shape.name, reason, publicops.MoleculeAutoCloseReason) + } + } +} + +// RunMoleculeAutoCloseIsOptInAndRuleBound pins the opt-in default and the +// three roots the auto-close must leave open: one a request did not ask for, +// an incomplete one, and an ordinary epic (explicit close-eligible work). +func RunMoleculeAutoCloseIsOptInAndRuleBound(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + + root, _, last := s.molecule("optin", types.TypeMolecule, false) + if res := s.closeStep(last, false); res.AutoClosedMolecule != nil { + t.Errorf("close without AutoCloseMolecule reported %s auto-closed", res.AutoClosedMolecule.ID) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("root %s status after an un-flagged close = %s, want open", root, status) + } + + root, _, last = s.molecule("plain", types.TypeEpic, false) + if res := s.closeStep(last, true); res.AutoClosedMolecule != nil { + t.Errorf("an ordinary epic auto-closed: %s", res.AutoClosedMolecule.ID) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("ordinary epic %s status = %s, want open", root, status) + } + + root, done, last := s.molecule("incomplete", types.TypeMolecule, false) + if res := s.closeStep(done, true); res.AutoClosedMolecule != nil { + t.Errorf("an incomplete molecule auto-closed: %s", res.AutoClosedMolecule.ID) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("incomplete molecule %s status = %s, want open (%s still open)", root, status, last) + } +} + +// RunMoleculeRecloseHealsAStrandedRoot pins the replay clause: an idempotent +// re-close of the last step still closes a root left open. +func RunMoleculeRecloseHealsAStrandedRoot(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root := fixture.IssuePrefix + "-stranded" + step := root + ".1" + s.issue(root, types.TypeMolecule, types.StatusOpen, false) + s.issue(step, types.TypeTask, types.StatusClosed, false) + s.edge(step, root, types.DepParentChild) + + res := s.closeStep(step, true) + if res.Changed { + t.Errorf("re-close of %s reported Changed", step) + } + if res.AutoClosedMolecule == nil || res.AutoClosedMolecule.ID != root { + t.Fatalf("re-close AutoClosedMolecule = %+v, want %s healed", res.AutoClosedMolecule, root) + } + if status, _ := s.row(root); status != string(types.StatusClosed) { + t.Errorf("stranded root %s status = %s, want closed", root, status) + } +} + +// RunMoleculeBatchCloseAutoClosesTheRootOnce pins the batch clause: a batch +// that closes the remaining steps closes the root once, on one outcome. +func RunMoleculeBatchCloseAutoClosesTheRootOnce(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root, _, last := s.molecule("batch", types.TypeMolecule, false) + other := root + ".3" + s.issue(other, types.TypeTask, types.StatusOpen, false) + s.edge(other, root, types.DepParentChild) + + res, err := fixture.BatchCloser.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: moleculeActor, + Items: []publicops.BatchCloseItem{{IssueID: last, Reason: "done"}, {IssueID: other, Reason: "done"}}, + AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + reported := 0 + for _, outcome := range res.Outcomes { + if outcome.Err != nil { + t.Fatalf("outcome %s: %v", outcome.IssueID, outcome.Err) + } + if outcome.AutoClosedMolecule != nil { + reported++ + if outcome.AutoClosedMolecule.ID != root { + t.Errorf("outcome %s auto-closed %s, want %s", outcome.IssueID, outcome.AutoClosedMolecule.ID, root) + } + } + } + if reported != 1 { + t.Errorf("the root was reported on %d outcomes, want exactly one", reported) + } + if status, _ := s.row(root); status != string(types.StatusClosed) { + t.Errorf("root %s status = %s, want closed", root, status) + } +} + +// RunMoleculeRefusedRootKeepsTheStepClose pins the refusal clause: a root its +// own close policy refuses (a live blocker) stays open and is reported in the +// policy's own sentence, and the step close it followed stands. +func RunMoleculeRefusedRootKeepsTheStepClose(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root, _, last := s.molecule("blocked", types.TypeMolecule, false) + blocker := root + "-blocker" + s.issue(blocker, types.TypeTask, types.StatusOpen, false) + s.edge(root, blocker, types.DepBlocks) + + res := s.closeStep(last, true) + if res.AutoClosedMolecule != nil { + t.Errorf("a blocked root auto-closed: %s", res.AutoClosedMolecule.ID) + } + if want := publicops.NewCloseBlockedError(root, []string{blocker}).Error(); res.MoleculeAutoCloseRefusal != want { + t.Errorf("MoleculeAutoCloseRefusal = %q, want the policy's own sentence %q", res.MoleculeAutoCloseRefusal, want) + } + if status, _ := s.row(last); status != string(types.StatusClosed) { + t.Errorf("step %s status = %s, want closed (the refusal must not undo it)", last, status) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("blocked root %s status = %s, want open", root, status) + } +} + +// RunMoleculeGuardedRootKeepsTheStepClose pins the guard clause: the root close +// is unforced, so a root a direct close would refuse — assigned to someone +// other than Actor, pinned, or a template — stays open and is reported in the +// guard's own sentence, on a single close and on a batch, and the step's close +// (and a batch's other items) stand. +func RunMoleculeGuardedRootKeepsTheStepClose(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + for _, guard := range []struct { + name string + apply func(*types.Issue) + // refusal is the guard's own error for root, whose sentence + // MoleculeAutoCloseRefusal reports. + refusal func(root string) error + }{ + {"assigned", func(root *types.Issue) { root.Assignee = "someone-else" }, func(root string) error { + return &publicops.CloseNotAssigneeError{IssueID: root, Assignee: "someone-else", Actor: moleculeActor} + }}, + {"pinned", func(root *types.Issue) { root.Pinned = true }, func(root string) error { + return &publicops.PinnedError{IssueID: root} + }}, + {"template", func(root *types.Issue) { root.IsTemplate = true }, func(root string) error { + return &publicops.TemplateReadOnlyError{IssueID: root} + }}, + } { + for _, batch := range []bool{false, true} { + name := guard.name + if batch { + name += "-batch" + } + root := fixture.IssuePrefix + "-guarded-" + name + last := root + ".2" + rootIssue := &types.Issue{ID: root, Title: "molecule " + root, IssueType: types.TypeMolecule, Status: types.StatusOpen, Priority: 2} + guard.apply(rootIssue) + if err := fixture.CreateIssue(ctx, rootIssue, moleculeActor); err != nil { + t.Fatalf("%s: seed %s: %v", name, root, err) + } + s.issue(root+".1", types.TypeTask, types.StatusClosed, false) + s.issue(last, types.TypeTask, types.StatusOpen, false) + s.edge(root+".1", root, types.DepParentChild) + s.edge(last, root, types.DepParentChild) + + var ( + autoClosed *publicops.Issue + refusal string + closed = []string{last} + ) + if batch { + other := root + "-other" + s.issue(other, types.TypeTask, types.StatusOpen, false) + closed = append(closed, other) + res, err := fixture.BatchCloser.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: moleculeActor, + Items: []publicops.BatchCloseItem{{IssueID: last, Reason: "done"}, {IssueID: other, Reason: "done"}}, + AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("%s: CloseBatch: %v", name, err) + } + for _, outcome := range res.Outcomes { + if outcome.Err != nil { + t.Fatalf("%s: outcome %s: %v", name, outcome.IssueID, outcome.Err) + } + if outcome.IssueID == last { + autoClosed, refusal = outcome.AutoClosedMolecule, outcome.MoleculeAutoCloseRefusal + } + } + } else { + res := s.closeStep(last, true) + autoClosed, refusal = res.AutoClosedMolecule, res.MoleculeAutoCloseRefusal + } + + if autoClosed != nil { + t.Errorf("%s: a guarded root auto-closed: %s", name, autoClosed.ID) + } + if want := guard.refusal(root).Error(); refusal != want { + t.Errorf("%s: MoleculeAutoCloseRefusal = %q, want the guard's own sentence %q", name, refusal, want) + } + for _, id := range closed { + if status, _ := s.row(id); status != string(types.StatusClosed) { + t.Errorf("%s: %s status = %s, want closed (the refusal must not undo it)", name, id, status) + } + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("%s: guarded root %s status = %s, want open", name, root, status) + } + } + } +} + +// RunMoleculeStepperClaimsTheNextReadyStepForTheActor pins the advance and its +// claim: the next ready step becomes in progress AND assigned to Actor; without +// AutoClaim nothing is written. +func RunMoleculeStepperClaimsTheNextReadyStepForTheActor(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root := fixture.IssuePrefix + "-advance" + a, b, c := root+".1", root+".2", root+".3" + s.issue(root, types.TypeMolecule, types.StatusOpen, false) + s.issue(a, types.TypeTask, types.StatusClosed, false) + s.issue(b, types.TypeTask, types.StatusOpen, false) + s.issue(c, types.TypeTask, types.StatusOpen, false) + for _, id := range []string{a, b, c} { + s.edge(id, root, types.DepParentChild) + } + s.edge(c, b, types.DepBlocks) + + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: a}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if res.MoleculeID != root || res.Claimed || res.NextStep == nil || res.NextStep.ID != b { + t.Fatalf("Advance without a claim = %+v, want next %s in %s, unclaimed", res, b, root) + } + if status, assignee := s.row(b); status != string(types.StatusOpen) || assignee != "" { + t.Errorf("%s after an unclaimed advance = %s/%q, want open and unassigned", b, status, assignee) + } + + res, err = fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: a, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance with a claim: %v", err) + } + if !res.Claimed || res.NextStep == nil || res.NextStep.ID != b || res.NextStep.Assignee != moleculeActor { + t.Fatalf("Advance with a claim = %+v, want %s claimed for %s", res, b, moleculeActor) + } + if status, assignee := s.row(b); status != string(types.StatusInProgress) || assignee != moleculeActor { + t.Errorf("%s after the claim = %s/%q, want in_progress assigned to %s", b, status, assignee, moleculeActor) + } + if status, _ := s.row(c); status != string(types.StatusOpen) { + t.Errorf("%s (blocked) status = %s, want open", c, status) + } +} + +// RunMoleculeStepperNeverTakesAStepAnotherActorHolds pins the refusal the claim +// carries: a ready step assigned to someone else is skipped for the next one, +// and left exactly as it was. +func RunMoleculeStepperNeverTakesAStepAnotherActorHolds(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root := fixture.IssuePrefix + "-held" + a, held, free := root+".1", root+".2", root+".3" + s.issue(root, types.TypeMolecule, types.StatusOpen, false) + s.issue(a, types.TypeTask, types.StatusClosed, false) + heldIssue := &types.Issue{ID: held, Title: "held", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2, Assignee: "someone-else"} + if err := fixture.CreateIssue(ctx, heldIssue, moleculeActor); err != nil { + t.Fatalf("seed %s: %v", held, err) + } + s.issue(free, types.TypeTask, types.StatusOpen, false) + for _, id := range []string{a, held, free} { + s.edge(id, root, types.DepParentChild) + } + + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: a, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || res.NextStep == nil || res.NextStep.ID != free { + t.Fatalf("Advance = %+v, want %s skipped (held) and %s claimed", res, held, free) + } + if status, assignee := s.row(held); status != string(types.StatusOpen) || assignee != "someone-else" { + t.Errorf("held step %s = %s/%q, want untouched (open, someone-else)", held, status, assignee) + } +} + +// RunMoleculeStepperAnswersCompleteAndOutsideAMolecule pins the two answers +// that claim nothing. +func RunMoleculeStepperAnswersCompleteAndOutsideAMolecule(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + orphan := fixture.IssuePrefix + "-orphan" + s.issue(orphan, types.TypeTask, types.StatusClosed, false) + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: orphan, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance(orphan): %v", err) + } + if res.MoleculeID != "" || res.NextStep != nil || res.Claimed || res.ClosedStep == nil || res.ClosedStep.ID != orphan { + t.Errorf("Advance(orphan) = %+v, want the step and no molecule", res) + } + + root := fixture.IssuePrefix + "-complete" + s.issue(root, types.TypeEpic, types.StatusOpen, false) + s.issue(root+".1", types.TypeTask, types.StatusClosed, false) + s.edge(root+".1", root, types.DepParentChild) + res, err = fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: root + ".1", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance(complete): %v", err) + } + if res.MoleculeID != root || !res.Complete || res.NextStep != nil || res.Claimed { + t.Errorf("Advance(complete) = %+v, want %s complete", res, root) + } + + if _, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: fixture.IssuePrefix + "-absent"}); !errors.Is(err, publicops.ErrNotFound) { + t.Errorf("Advance(absent) = %v, want ErrNotFound", err) + } +} + +// RunMoleculeStepperClaimsAWispStep pins the wisp plane: a wisp molecule's next +// step is claimed for the actor too. +func RunMoleculeStepperClaimsAWispStep(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root, done, last := s.molecule("wisp", types.TypeMolecule, true) + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: done, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if res.MoleculeID != root || !res.Claimed || res.NextStep == nil || res.NextStep.ID != last { + t.Fatalf("Advance over a wisp molecule = %+v, want %s claimed", res, last) + } + if status, assignee := s.row(last); status != string(types.StatusInProgress) || assignee != moleculeActor { + t.Errorf("wisp step %s = %s/%q, want in_progress assigned to %s", last, status, assignee, moleculeActor) + } +} diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index 723cb49b1..d8d4e273a 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -32,6 +32,7 @@ go_test( srcs = [ "attach_test.go", "e2e_cli_test.go", + "e2e_mol_test.go", "e2e_remote_roles_test.go", "open_test.go", "per_open_credential_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 99a9a9c8b..e7a74b0d7 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -214,7 +214,6 @@ shapes: - id: mc-ready-mol source: mc skills/pr-review (bd ready --mol ) args: [ready, --mol, e2e-mol, --json] - pending: {slice: "S16", reason: "ready --mol loads molecule steps off a raw path: step rows lose assignee and timestamps (molecule reads over roles)"} - id: mc-ready-parent source: mc skills/pr-review (bd ready --parent ) args: [ready, --parent, e2e-ep, --json] @@ -354,20 +353,16 @@ shapes: - id: mc-mol-current source: mc worker prompts (bd mol current, ~38; inferred from in_progress work) args: [mol, current, --json] - pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} - id: mc-mol-current-text source: mc gc-role-worker template (bd mol current, text) args: [mol, current] compare: text - pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} - id: mc-mol-current-id source: mc prompts (bd mol current ) args: [mol, current, e2e-mol, --json] - pending: {slice: "S16", reason: "mol current refuses (B3 #4)"} - id: mc-mol-progress source: mc prompts (bd mol progress ) args: [mol, progress, e2e-mol, --json] - pending: {slice: "S16", reason: "mol progress refuses (B3 #4)"} # ---------------------------------------------------------------- config / probes / local - id: gc-config-get-types @@ -445,7 +440,6 @@ shapes: args: [create, --json, "gc child", -t, task, --parent, e2e-ep] compare: exit verify: [[show, --json, e2e-ep.3]] - pending: {slice: "S19", reason: "create --parent calls GetNextChildID (B3 #8)"} - id: gc-create-ephemeral source: gc bdstore.go CreateWithStorage (--ephemeral) args: [create, --json, "gc wisp", -t, task, --id, e2e-wisp-n2, --ephemeral, --metadata, '{"gc.routed_to":"pool-a"}'] diff --git a/backend/http/e2e_mol_test.go b/backend/http/e2e_mol_test.go new file mode 100644 index 000000000..2d511e8cb --- /dev/null +++ b/backend/http/e2e_mol_test.go @@ -0,0 +1,305 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestE2E_MoleculesOverHTTP drives the molecule workflow a gc worker runs — +// `bd create --parent`, `bd mol current`, `bd mol progress`, `bd ready --mol`, +// `bd close --continue` and the molecule auto-close after the last step — +// through the real `bd` binary against an in-process `bd serve` (slices S16 +// and S19). +// +// Every one of these used to fail or, worse, answer silently over http. Each +// now has one library entry the server runs: the create role mints the child +// id, the molecule view (issueops.ViewMolecule) answers the reads, the +// close runs the auto-close in its own transaction (auto_close_molecule), and +// `close --continue` is the advanceMolecule operation, whose claim sets the +// assignee and never takes a step someone else holds. +func TestE2E_MoleculesOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + workspace := t.TempDir() + + bd := func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, workspace, nil, append([]string{"--actor", "alice"}, args...)...) + } + mustBD := func(args ...string) bdResult { + t.Helper() + r := bd(args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + return r + } + status := func(id string) string { + t.Helper() + return firstJSONField(t, mustBD("show", id, "--json").stdout, "status") + } + + mustBD("connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json") + + root := firstJSONField(t, mustBD("create", "Patrol molecule", "-t", "molecule", "--json").stdout, "id") + if root == "" { + t.Fatal("bd create (molecule root) produced no id") + } + + // S19: the server mints .; the client no longer asks a raw + // GetNextChildID the http backend does not serve. + step1 := firstJSONField(t, mustBD("create", "Step one", "--parent", root, "--json").stdout, "id") + step2 := firstJSONField(t, mustBD("create", "Step two", "--parent", root, "--json").stdout, "id") + step3 := firstJSONField(t, mustBD("create", "Step three", "--parent", root, "--json").stdout, "id") + if step1 != root+".1" || step2 != root+".2" || step3 != root+".3" { + t.Fatalf("create --parent minted %q, %q and %q, want %q, %q and %q", step1, step2, step3, root+".1", root+".2", root+".3") + } + // An explicit --id beside --parent is still refused locally, before any mint. + if r := bd("create", "Both", "--parent", root, "--id", root+".9", "--json"); r.code == 0 { + t.Errorf("create --parent --id succeeded; want the existing refusal: %s", r.stdout) + } + + mustBD("dep", "add", step2, step1, "--json") + mustBD("dep", "add", step3, step1, "--json") + mustBD("update", step1, "-s", "in_progress", "-a", "alice", "--json") + // Someone else holds step two: the advance must skip it, not take it. + mustBD("update", step2, "-a", "bob", "--json") + + // S16: mol current with an id is a non-empty answer whose current step is + // the in-progress one. + current := decodeMolecules(t, mustBD("mol", "current", root, "--json").stdout) + if len(current) != 1 || current[0].MoleculeID != root { + t.Fatalf("mol current %s = %+v, want the one molecule %s", root, current, root) + } + if current[0].CurrentStep == nil || current[0].CurrentStep.ID != step1 || current[0].Total != 3 { + t.Fatalf("mol current %s current=%v total=%d, want current %s of 3", root, current[0].CurrentStep, current[0].Total, step1) + } + // The step is the FULL row: the raw dependents read over http is getIssue's + // shallow projection, which zeroes the assignee a worker prompt reads. + if got := current[0].CurrentStep.Assignee; got != "alice" { + t.Errorf("mol current %s current step assignee = %q, want alice (full step row)", root, got) + } + + progress := mustBD("mol", "progress", root, "--json").stdout + if n := jsonIntField(t, progress, "total"); n != 3 { + t.Errorf("mol progress total = %d, want 3: %s", n, progress) + } + if id := firstJSONField(t, progress, "current_step_id"); id != step1 { + t.Errorf("mol progress current_step_id = %q, want %q: %s", id, step1, progress) + } + + // mol show reads through the same library entry (issueops.LoadMolecule). + var shown struct { + Root struct{ ID string } `json:"root"` + Issues []struct { + ID string `json:"id"` + Assignee string `json:"assignee"` + } `json:"issues"` + } + showOut := mustBD("mol", "show", root, "--json").stdout + if err := json.Unmarshal([]byte(strings.TrimSpace(showOut)), &shown); err != nil { + t.Fatalf("parse mol show: %v\n%s", err, showOut) + } + if shown.Root.ID != root || len(shown.Issues) != 4 { + t.Errorf("mol show %s = root %q with %d issues, want %s and 4 (root + 3 steps): %s", root, shown.Root.ID, len(shown.Issues), root, showOut) + } + + t.Run("mol current without an id", func(t *testing.T) { + r := bd("mol", "current", "--json") + // Whatever the backend can do, it must not answer an EMPTY list for an + // agent with a molecule in progress: that was the swallowed refusal. + if r.code == 0 && len(decodeMolecules(t, r.stdout)) == 0 { + t.Fatalf("mol current (no id) answered an empty list while %s is in progress: stderr=%s", step1, r.stderr) + } + if r.code != 0 { + t.Fatalf("mol current (no id) failed (exit %d): %s stdout=%s", r.code, r.stderr, r.stdout) + } + got := decodeMolecules(t, r.stdout) + if len(got) != 1 || got[0].MoleculeID != root { + t.Errorf("mol current (no id) = %+v, want molecule %s", got, root) + } + }) + + // ready --mol: the step rows are the full rows (the corpus mark + // mc-ready-mol was the shallow projection dropping the assignee). + var ready struct { + Steps []struct { + Issue struct { + ID string `json:"id"` + Assignee string `json:"assignee"` + } `json:"issue"` + } `json:"steps"` + } + readyOut := mustBD("ready", "--mol", root, "--json").stdout + if err := json.Unmarshal([]byte(strings.TrimSpace(readyOut)), &ready); err != nil { + t.Fatalf("parse ready --mol: %v\n%s", err, readyOut) + } + sawStep1 := false + for _, step := range ready.Steps { + if step.Issue.ID == step1 { + sawStep1 = true + if step.Issue.Assignee != "alice" { + t.Errorf("ready --mol %s step %s assignee = %q, want alice (full step row)", root, step1, step.Issue.Assignee) + } + } + } + if !sawStep1 { + t.Errorf("ready --mol %s did not list the in-progress step %s: %s", root, step1, readyOut) + } + + // close --continue: step two is first in line but bob holds it, so the + // advance claims step three for alice — a real claim, assignee set. + cont := mustBD("close", step1, "--continue", "--json") + var contBody struct { + Continue struct { + NextStep *struct{ ID string } `json:"next_step"` + AutoAdvanced bool `json:"auto_advanced"` + MoleculeID string `json:"molecule_id"` + } `json:"continue"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(cont.stdout)), &contBody); err != nil { + t.Fatalf("parse close --continue output: %v\n%s", err, cont.stdout) + } + if !contBody.Continue.AutoAdvanced || contBody.Continue.NextStep == nil || contBody.Continue.NextStep.ID != step3 || + contBody.Continue.MoleculeID != root { + t.Errorf("close --continue = %s, want auto-advance to %s in %s (skipping %s, held by bob)", cont.stdout, step3, root, step2) + } + if got := status(step3); got != "in_progress" { + t.Errorf("%s status after close --continue = %q, want in_progress", step3, got) + } + if got := firstJSONField(t, mustBD("show", step3, "--json").stdout, "assignee"); got != "alice" { + t.Errorf("%s assignee after close --continue = %q, want alice (a claim, not a status flip)", step3, got) + } + if got := firstJSONField(t, mustBD("show", step2, "--json").stdout, "assignee"); got != "bob" || status(step2) != "open" { + t.Errorf("%s after close --continue = %s/%q, want open and still bob's", step2, status(step2), got) + } + if got := status(root); got == "closed" { + t.Fatalf("molecule %s closed with %s still open", root, step2) + } + + // Closing the last steps auto-closes the molecule root. + mustBD("close", step3) + if got := status(root); got == "closed" { + t.Fatalf("molecule %s closed with %s still open", root, step2) + } + last := mustBD("close", step2, "--force") + if strings.Contains(last.stderr, "could not") { + t.Errorf("close %s warned: %s", step2, last.stderr) + } + if got := status(root); got != "closed" { + t.Errorf("molecule %s status after its last step closed = %q, want closed (auto-close): stdout=%s stderr=%s", + root, got, last.stdout, last.stderr) + } +} + +type e2eMolecule struct { + MoleculeID string `json:"molecule_id"` + Total int `json:"total"` + CurrentStep *struct { + ID string `json:"id"` + Assignee string `json:"assignee"` + } `json:"current_step"` +} + +func decodeMolecules(t *testing.T, out string) []e2eMolecule { + t.Helper() + var got []e2eMolecule + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &got); err != nil { + t.Fatalf("parse mol current output: %v\n%s", err, out) + } + return got +} + +// TestE2E_MoleculeCloseAgainstAnOlderServer is `bd close` on a molecule step +// against a server that advertises neither issues.close.autoCloseMolecule nor +// issues.advanceMolecule (an older bd serve, or one whose orchestrator owns +// the molecule lifecycle). Neither gap may be silent: +// +// - close --continue refuses BEFORE closing, so the step is not left closed +// with the next step unclaimed; +// - a plain close goes out without the auto-close request and names the +// molecule it did not auto-close on stderr. +func TestE2E_MoleculeCloseAgainstAnOlderServer(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr, _ := startMaskingProxy(t, startE2EServer(t), func(body map[string]any) { + caps, _ := body["capabilities"].([]any) + kept := caps[:0] + for _, c := range caps { + if c != "issues.close.autoCloseMolecule" && c != "issues.advanceMolecule" { + kept = append(kept, c) + } + } + body["capabilities"] = kept + }) + workspace := t.TempDir() + bd := func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, workspace, nil, append([]string{"--actor", "alice"}, args...)...) + } + mustBD := func(args ...string) bdResult { + t.Helper() + r := bd(args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + return r + } + status := func(id string) string { + t.Helper() + return firstJSONField(t, mustBD("show", id, "--json").stdout, "status") + } + + mustBD("connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json") + root := firstJSONField(t, mustBD("create", "Old-server molecule", "-t", "molecule", "--json").stdout, "id") + step := firstJSONField(t, mustBD("create", "First step", "--parent", root, "--json").stdout, "id") + last := firstJSONField(t, mustBD("create", "Last step", "--parent", root, "--json").stdout, "id") + epic := firstJSONField(t, mustBD("create", "Ordinary epic", "-t", "epic", "--json").stdout, "id") + child := firstJSONField(t, mustBD("create", "Only epic child", "--parent", epic, "--json").stdout, "id") + + cont := bd("close", step, "--continue", "--json") + if cont.code == 0 { + t.Fatalf("close --continue succeeded against a server without issues.advanceMolecule: %s", cont.stdout) + } + if !strings.Contains(cont.stderr+cont.stdout, "issues.advanceMolecule") || !strings.Contains(cont.stderr+cont.stdout, "nothing was closed") { + t.Errorf("close --continue refusal does not name the missing operation: stdout=%s stderr=%s", cont.stdout, cont.stderr) + } + if got := status(step); got == "closed" { + t.Fatalf("close --continue refused but %s is closed: the refusal came after the close", step) + } + + // The notice is owed only where the close left an auto-closing molecule + // complete with its root open: not for a step whose molecule still has an + // open step, and not under an ordinary epic, which never auto-closes. + for _, id := range []string{step, child} { + if r := mustBD("close", id, "--json"); strings.Contains(r.stderr, "issues.close.autoCloseMolecule") { + t.Errorf("close of %s completed no auto-closing molecule but printed the unserved auto-close notice: stderr=%s", id, r.stderr) + } + } + // The close that completes the molecule names its root, and so does an + // idempotent re-close while the root stays open. + for _, attempt := range []string{"close", "re-close"} { + r := mustBD("close", last, "--json") + if !strings.Contains(r.stderr, "completed molecule "+root) || !strings.Contains(r.stderr, "issues.close.autoCloseMolecule") { + t.Errorf("%s of %s against a server without the auto-close gave no notice naming %s: stderr=%s", attempt, last, root, r.stderr) + } + } + for _, id := range []string{step, last, child} { + if got := status(id); got != "closed" { + t.Errorf("%s status = %q, want closed", id, got) + } + } + if got := status(root); got == "closed" { + t.Errorf("molecule %s closed although the server was not asked to auto-close it", root) + } + + // The root the notice names closes by hand; then nothing is owed. + mustBD("close", root, "--json") + if r := mustBD("close", last, "--json"); strings.Contains(r.stderr, "issues.close.autoCloseMolecule") { + t.Errorf("re-close of %s under the closed %s printed the unserved auto-close notice: stderr=%s", last, root, r.stderr) + } +} diff --git a/backend/http/e2e_remote_roles_test.go b/backend/http/e2e_remote_roles_test.go index 70aba8e36..397e5facd 100644 --- a/backend/http/e2e_remote_roles_test.go +++ b/backend/http/e2e_remote_roles_test.go @@ -691,9 +691,18 @@ func TestE2E_AdvertisedExternalPolicyIsPassedThrough(t *testing.T) { t.Errorf("claimNext requests = %d, by-id claims = %d; want one atomic claim-next and no by-id claim: %v", claimNexts, byID, counter.snapshot()) } f.mustRun(nil, "close", other, "--json") - for _, p := range counter.snapshot() { + // The close policy check is the client-side policy's last chance: it runs + // before the close request, so only the reads up to that request count + // here. The molecule auto-close is not a client read any more; it rides on + // the close request itself (auto_close_molecule) and the server performs + // it inside the close. + paths := counter.snapshot() + for _, p := range paths { + if strings.HasPrefix(p, "POST /v0/beads/issues:batchClose") { + break + } if strings.HasPrefix(p, "GET /v0/beads/dependencies") { - t.Errorf("client-side policy ran against a server advertising policy.external_dependencies: %s in %v", p, counter.snapshot()) + t.Errorf("client-side policy ran against a server advertising policy.external_dependencies: %s in %v", p, paths) break } } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7c493dc27..bcf823bf2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -249,6 +249,7 @@ go_library( "mol_progress.go", "mol_proxied_server.go", "mol_ready_gated.go", + "mol_roles.go", "mol_seed.go", "mol_show.go", "mol_squash.go", @@ -997,6 +998,7 @@ go_test( "mol_ready_gated_hint_test.go", "mol_ready_gated_proxied_integration_test.go", "mol_ready_gated_test.go", + "mol_route_parity_integration_test.go", "mol_show_proxied_integration_test.go", "mol_squash_embedded_test.go", "mol_squash_proxied_integration_test.go", diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 07eec5fa2..0d8760c0c 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -37,7 +37,16 @@ the fallback anywhere, or =0 to disable it entirely. When closing multiple issues, provide one --reason for all IDs or repeat --reason once per ID. Reasons map positionally: the first --reason applies to the first ID, the second --reason to the second ID, regardless of where -the flags appear in the command line.`, +the flags appear in the command line. + +Closing a molecule's last open step also closes the molecule's root, with +the reason "all steps complete", when the root is a poured molecule, a wisp, +or an epic labeled "template"; any other epic stays open as close-eligible +work. A root that would refuse its own close (an open blocker, a pin, another +assignee) stays open with a warning, and the step's close stands. Under +--json, the step's entry names the closed root (auto_closed_molecule) or the +refusal (molecule_auto_close_refusal). --continue claims the next ready step +in the molecule for you; add --no-auto to only show it.`, // Refuse a missing ID in argument validation, before root's // PersistentPreRunE can open the store, migrate, or auto-import // (bd-m00pb); see updateCmd for the full rationale. @@ -153,6 +162,21 @@ the flags appear in the command line.`, postCloseStore = results[0].Store } + // --continue is a second operation after the close commits. Against a + // server that cannot run it (no issues.advanceMolecule), or whose + // handshake cannot be read, refuse BEFORE closing: refusing only at the + // advance would leave the step closed and the next step unclaimed, the + // half-done state --continue exists to avoid. + if continueFlag { + served, err := storage.ServesMoleculeAdvance(ctx, postCloseStore) + if err != nil { + return HandleErrorRespectJSON("--continue could not read whether the server serves issues.advanceMolecule: %v; nothing was closed", err) + } + if !served { + return HandleErrorRespectJSON("--continue needs the server's issues.advanceMolecule operation, which this server does not advertise; nothing was closed (close without --continue, then claim the next step with bd update --claim)") + } + } + // THE BATCH. The CLI's own close policy runs first and read-only, then // every id it passed goes to the BatchCloser role as one request per // store: one transaction, one Dolt commit over N ids, and an id the @@ -165,11 +189,14 @@ the flags appear in the command line.`, session, force, postCloseStore, closeClaimNextRequest(claimNext, continueFlag)) // Report and follow up on every argument, in the order it was typed. - closedIssues := []*types.Issue{} + closedIssues := []closedIssueJSON{} closedCount := 0 alreadyClosed := 0 var failures []closeIDFailure firstSettledID := "" + // The roots noticeUnservedMoleculeAutoClose already named, so a batch + // that closes several steps of one molecule names it once. + unservedRoots := map[string]bool{} for i, id := range resolvedIDs { res := outcomes[i] @@ -215,21 +242,9 @@ the flags appear in the command line.`, // Exit stays 0. alreadyClosed++ - // Molecule auto-close is itself a retry-safe, fully state-derived - // post-close contract, so it must replay on an already-closed re-close - // just like the contracts above. If the final step's real close - // persisted but its molecule auto-close did not (a crash between the two - // commits, or the root CloseIssue failing with only a warning), this - // idempotent re-close is the ONLY thing that re-drives it — otherwise the - // molecule root is stranded open forever. autoCloseCompletedMolecule - // early-returns unless the root is genuinely open, auto-close-eligible, - // and complete, so it heals only that case and reintroduces none of the - // suppressed real-close side effects (no audit, no closed→closed on the - // step). Register the store when it actually closed the root so the - // pending-commit sweep persists it — closedCount==0 would not commit. - if molID := autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session); molID != "" { - mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) - } + // The molecule auto-close is state-derived, so the library + // replays it on an already-closed re-close too (healing a root a + // crash left open); reportMoleculeAutoClose below covers both arms. } else { mutatedStores[activeStore] = append(mutatedStores[activeStore], id) @@ -241,12 +256,17 @@ the flags appear in the command line.`, audit.LogFieldChange(id, "status", oldStatus, "closed", currentActor(), reason) closedCount++ - - // Auto-close parent molecule if all steps are now complete. - // Runs against the same store the step was closed in. - autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session) } + // The molecule root this close completed, closed by the library in + // the batch's own transaction. Register its store so the + // pending-commit sweep persists it even when the step itself was + // an already-closed no-op. + if molID := reportMoleculeAutoClose(res.AutoClosedMolecule, res.MoleculeAutoCloseRefusal); molID != "" { + mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) + } + noticeUnservedMoleculeAutoClose(ctx, activeStore, id, unservedRoots) + // First id this command settled as closed — a real close or an // already-closed no-op both "touch" it. Drives the retry-safe last-touched // contract below so a re-close still points default-target commands at it. @@ -266,7 +286,7 @@ the flags appear in the command line.`, if jsonOutput { if closedIssue != nil { - closedIssues = append(closedIssues, closedIssue) + closedIssues = append(closedIssues, newClosedIssueJSON(closedIssue, res.AutoClosedMolecule, res.MoleculeAutoCloseRefusal)) } } else { debug.PrintNormal("%s Closed %s: %s\n", ui.RenderPass("✓"), formatFeedbackID(id, issueTitleOrEmpty(issue)), reason) @@ -313,11 +333,11 @@ the flags appear in the command line.`, if continueFlag && len(resolvedIDs) == 1 && closedForCommand { autoClaim := !noAuto - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, currentActor()) + result, err := advanceMoleculeDirect(ctx, postCloseStore, resolvedIDs[0], autoClaim) if err != nil { fmt.Fprintf(os.Stderr, "Warning: could not advance to next step: %v\n", err) } else if result != nil { - // Mirror --claim-next: when AdvanceToNextStep auto-claims the + // Mirror --claim-next: when the molecule advance auto-claims the // next step, update .beads/last-touched so subsequent default- // target commands (e.g. bare `bd update`, `bd close`) target // it. Without this, last-touched stays pointed at the just- @@ -742,74 +762,88 @@ func closeBeadGateGetter(gateStore storage.DoltStorage) issueGetter { return routedBeadGateGetter{localStore: gateStore} } -// autoCloseCompletedMolecule checks if closing a step completed an auto-closing -// parent molecule, and if so, closes the molecule root. Ordinary epics remain -// open when all children finish so they can become explicitly close-eligible -// instead of being closed as a side effect of the final child close. It returns -// the molecule root ID when it actually closed the root (and "" otherwise) so a -// caller that did not otherwise mutate the store — an already-closed re-close in -// particular — can register the store for the pending-commit sweep. The check is -// fully state-derived and idempotent: it early-returns unless the root is open, -// auto-close-eligible, and has all steps complete, so re-invoking it never -// double-closes or reintroduces side effects. -func autoCloseCompletedMolecule(ctx context.Context, s storage.DoltStorage, closedStepID, actorName, session string) string { - moleculeID := findParentMolecule(ctx, s, closedStepID) - if moleculeID == "" { - return "" // Not part of a molecule +// reportMoleculeAutoClose presents what the library's molecule auto-close did +// for one close (issueops.CloseRequest.AutoCloseMolecule): the root it closed, +// or why a completed root stayed open. It returns the closed root's id, or "". +func reportMoleculeAutoClose(root *types.Issue, refusal string) string { + if refusal != "" { + fmt.Fprintf(os.Stderr, "Warning: could not auto-close completed molecule: %s\n", refusal) } - - // Check if molecule root is already closed - root, err := s.GetIssue(ctx, moleculeID) - if err != nil || root == nil || root.Status == types.StatusClosed || !shouldAutoCloseCompletedRoot(root) { + if root == nil { return "" } - - // Load progress to check completion - progress, err := getMoleculeProgress(ctx, s, moleculeID) - if err != nil { - return "" // Best effort — don't fail the close - } - - if progress.Completed < progress.Total { - return "" // Not all steps complete yet - } - - // All steps complete — auto-close the molecule root - if err := s.CloseIssue(ctx, moleculeID, "all steps complete", actorName, session); err != nil { - fmt.Fprintf(os.Stderr, "Warning: could not auto-close completed molecule %s: %v\n", moleculeID, err) - return "" - } - if !jsonOutput { - debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(moleculeID, root.Title)) + debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(root.ID, root.Title)) } - return moleculeID + return root.ID } -// shouldAutoCloseCompletedRoot returns true for molecule roots that should -// auto-close when their final step closes. Regular epics stay open and become -// explicit close-eligible work, while ephemeral wisps, template-driven -// molecules, and molecule-type coordination roots keep their cleanup behavior. -func shouldAutoCloseCompletedRoot(root *types.Issue) bool { - if root == nil { - return false - } +// closedIssueJSON is one entry of `bd close --json`'s list of closed issues: +// the closed issue's own members, and beside them what its close did to the +// molecule it completed (issueops.CloseResult.AutoClosedMolecule and +// MoleculeAutoCloseRefusal, under the HTTP API's BatchCloseOutcome names). +// Beside, not around: every entry still decodes as the issue it always was, +// and both members are absent unless the close completed a molecule. A batch +// reports each root once, on the first entry under it. +type closedIssueJSON struct { + *types.Issue + AutoClosedMolecule *types.Issue `json:"auto_closed_molecule,omitempty"` + MoleculeAutoCloseRefusal string `json:"molecule_auto_close_refusal,omitempty"` +} - if root.IssueType == types.TypeMolecule || root.Ephemeral { - return true +// newClosedIssueJSON is closed's entry. The auto-closed root drops its +// dependency records, as the closed issue does: `bd close` never prints them. +func newClosedIssueJSON(closed, autoClosedMolecule *types.Issue, refusal string) closedIssueJSON { + if autoClosedMolecule != nil { + autoClosedMolecule.Dependencies = nil } + return closedIssueJSON{Issue: closed, AutoClosedMolecule: autoClosedMolecule, MoleculeAutoCloseRefusal: refusal} +} - if root.IssueType != types.TypeEpic { - return false +// noticeUnservedMoleculeAutoClose names, on stderr, the molecule a close left +// complete but could not ask to auto-close. Against a server that does not +// advertise issues.close.autoCloseMolecule (an older bd serve, or one whose +// orchestrator owns the molecule lifecycle) the close goes out without the +// request (storage.ServesMoleculeAutoClose), so a close after which +// issueops.CompletedMolecule answers a root must say so: the root's auto-close +// is that server's to run, never silently skipped. A step under an ordinary +// epic, under a molecule with a step still open, or under a root already +// closed prints nothing. An already-closed step whose molecule is complete +// with its root open names the root again, as the served route replays its +// auto-close. noted, when not nil, holds the roots this command already +// named, so a batch names each once. A store that serves the auto-close prints +// nothing. +func noticeUnservedMoleculeAutoClose(ctx context.Context, s storage.DoltStorage, stepID string, noted map[string]bool) { + if s == nil || storage.ServesMoleculeAutoClose(ctx, s) { + return } - - for _, label := range root.Labels { - if label == BeadsTemplateLabel { - return true - } + const token = "issues.close.autoCloseMolecule" + reader, err := issueops.NewMoleculeReader(s) + var root *types.Issue + if err == nil { + root, err = issueops.CompletedMolecule(ctx, reader, stepID) } + if err != nil { + fmt.Fprintf(os.Stderr, "note: %s was closed without its molecule auto-close (the server does not advertise %s), and its molecule could not be read: %v\n", stepID, token, err) + return + } + if root == nil || noted[root.ID] { + return + } + if noted != nil { + noted[root.ID] = true + } + fmt.Fprintf(os.Stderr, "note: closing %s completed molecule %s, which this close did not auto-close: the server does not advertise %s, so the molecule's auto-close is the server's to run (bd close %s closes it)\n", stepID, root.ID, token, root.ID) +} - return false +// advanceMoleculeDirect runs `bd close --continue` on the direct route through +// the store's MoleculeStepper role. +func advanceMoleculeDirect(ctx context.Context, s storage.DoltStorage, closedStepID string, autoClaim bool) (*ContinueResult, error) { + roles, err := directMoleculeRoles(s) + if err != nil { + return nil, err + } + return advanceMolecule(ctx, roles, closedStepID, autoClaim) } // resolveReasonFile resolves the --reason-file flag for `bd close`. diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index 79a91c59a..0e0c82955 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -133,12 +133,18 @@ func closeDirectBatches(items []closeDirectItem) []closeDirectBatch { // // claimNext rides on the batch of claimStore and no other, because --claim-next // hands out ONE claim however many stores the ids spanned. -func closeDirectRequest(batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { +func closeDirectRequest(ctx context.Context, batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { request := issueops.CloseBatchRequest{ Actor: currentActor(), Items: make([]issueops.BatchCloseItem, 0, len(batch.items)), Session: session, Force: force, + // bd closes a molecule's root when its last step closes; the library + // does it inside the batch's transaction. Opt-in at the library so an + // orchestrator with its own molecule lifecycle is not doubled, and + // asked for only where the store serves it (a remote server that does + // not advertise it owns the molecule's auto-close). + AutoCloseMolecule: storage.ServesMoleculeAutoClose(ctx, batch.store), } if claimNext != nil && batch.store == claimStore { request.ClaimNext = claimNext @@ -164,7 +170,7 @@ func closeDirectRun(ctx context.Context, batches []closeDirectBatch, argCount in var claimed *types.IssueWithCounts for _, batch := range batches { - result, err := closeDirectCloseBatch(ctx, batch.store, closeDirectRequest(batch, session, force, claimStore, claimNext)) + result, err := closeDirectCloseBatch(ctx, batch.store, closeDirectRequest(ctx, batch, session, force, claimStore, claimNext)) if err != nil { for _, item := range batch.items { refused := issueops.CloseOutcome{IssueID: item.id, Err: err} diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index 2c87dbac0..0fe7f968c 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // ===== Close-specific test helpers ===== @@ -56,6 +57,80 @@ func bdDepAdd(t *testing.T, bd, dir string, args ...string) { } } +// closeJSONEntry is one entry of `bd close --json`, decoded as a consumer that +// knows the molecule auto-close members does: the closed issue's own members +// at the top level, with the members beside them. +type closeJSONEntry struct { + types.Issue + AutoClosedMolecule *types.Issue `json:"auto_closed_molecule"` + MoleculeAutoCloseRefusal *string `json:"molecule_auto_close_refusal"` +} + +// decodeCloseJSON decodes the array of closed issues `bd close --json` prints +// on stdout, and requires one entry per id in ids, in order, each closed. +func decodeCloseJSON(t *testing.T, stdout string, ids []string) []closeJSONEntry { + t.Helper() + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("no JSON array in bd close --json output:\n%s", stdout) + } + var entries []closeJSONEntry + if err := json.Unmarshal([]byte(stdout[start:]), &entries); err != nil { + t.Fatalf("parse bd close --json: %v\n%s", err, stdout[start:]) + } + if len(entries) != len(ids) { + t.Fatalf("bd close --json has %d entries, want %d (%v):\n%s", len(entries), len(ids), ids, stdout[start:]) + } + for i, entry := range entries { + if entry.ID != ids[i] || entry.Status != types.StatusClosed { + t.Fatalf("entry %d = %s (%s), want %s closed:\n%s", i, entry.ID, entry.Status, ids[i], stdout[start:]) + } + } + return entries +} + +// requireCloseJSONReportsAutoClosedMolecule asserts a `bd close --json` of +// stepIDs, in order, whose close completed the molecule rootID: the first +// entry carries the closed root as auto_closed_molecule, and no entry carries +// the root again or a refusal. +func requireCloseJSONReportsAutoClosedMolecule(t *testing.T, stdout string, stepIDs []string, rootID string) { + t.Helper() + entries := decodeCloseJSON(t, stdout, stepIDs) + root := entries[0].AutoClosedMolecule + if root == nil { + t.Fatalf("the close that completed molecule %s does not report it as auto_closed_molecule:\n%s", rootID, stdout) + } + if root.ID != rootID || root.Status != types.StatusClosed || root.CloseReason != "all steps complete" { + t.Errorf("auto_closed_molecule = %s (%s, %q), want %s closed with %q", root.ID, root.Status, root.CloseReason, rootID, "all steps complete") + } + for i, entry := range entries { + if i > 0 && entry.AutoClosedMolecule != nil { + t.Errorf("entry %d (%s) reports molecule %s again: a batch reports each root once", i, entry.ID, entry.AutoClosedMolecule.ID) + } + if entry.MoleculeAutoCloseRefusal != nil { + t.Errorf("entry %d (%s) reports a refusal for a root that closed: %q", i, entry.ID, *entry.MoleculeAutoCloseRefusal) + } + } +} + +// requireCloseJSONReportsMoleculeRefusal asserts a `bd close --json` of +// stepID whose close completed the molecule rootID but could not close it: +// the entry carries no auto-closed root and reports the refusal as want, the +// refusing guard's own sentence, on every route. +func requireCloseJSONReportsMoleculeRefusal(t *testing.T, stdout, stepID, rootID string, want error) { + t.Helper() + entry := decodeCloseJSON(t, stdout, []string{stepID})[0] + if entry.AutoClosedMolecule != nil { + t.Errorf("a refused root is reported as auto-closed: %s", entry.AutoClosedMolecule.ID) + } + if entry.MoleculeAutoCloseRefusal == nil { + t.Fatalf("the close that completed molecule %s does not report why its root stayed open:\n%s", rootID, stdout) + } + if got := *entry.MoleculeAutoCloseRefusal; got != want.Error() { + t.Errorf("molecule_auto_close_refusal for %s:\n got %q\nwant %q", rootID, got, want.Error()) + } +} + // ===== Close tests ===== // The TestEmbeddedCloseBasic*, TestEmbeddedCloseGuardsAndEpics and @@ -648,6 +723,44 @@ func TestEmbeddedCloseAlreadyClosedClaimNextAndMolecule(t *testing.T) { } +// TestEmbeddedCloseJSONReportsMoleculeAutoClose pins what `bd close --json` +// says about the molecule a close completed, on the direct route: the root it +// closed, once per batch, or why the root stayed open. +// TestProxiedServerClose2B's json_reports_* subtests are its proxied twins. +func TestEmbeddedCloseJSONReportsMoleculeAutoClose(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + + t.Run("auto_closed_root_reported_once_per_batch", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "jm") + root := bdCreate(t, bd, dir, "JSON molecule root", "--type", "epic", "--labels", "template") + step1 := bdCreate(t, bd, dir, "JSON molecule step one", "--type", "task", "--parent", root.ID) + step2 := bdCreate(t, bd, dir, "JSON molecule step two", "--type", "task", "--parent", root.ID) + step3 := bdCreate(t, bd, dir, "JSON molecule step three", "--type", "task", "--parent", root.ID) + bdClose(t, bd, dir, step1.ID) + + out := bdClose(t, bd, dir, "--json", step2.ID, step3.ID) + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{step2.ID, step3.ID}, root.ID) + }) + + t.Run("refused_root_reported", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "jr") + root := bdCreate(t, bd, dir, "JSON held molecule root", "--type", "epic", "--labels", "template", "--assignee", "someone-else") + step := bdCreate(t, bd, dir, "JSON held molecule step", "--type", "task", "--parent", root.ID) + + out := bdClose(t, bd, dir, "--actor", "json-closer", "--json", step.ID) + requireCloseJSONReportsMoleculeRefusal(t, out, step.ID, root.ID, + &issueops.CloseNotAssigneeError{IssueID: root.ID, Assignee: "someone-else", Actor: "json-closer"}) + if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusOpen { + t.Errorf("a root assigned to another actor must stay open, got %s", got.Status) + } + }) +} + // TestEmbeddedCloseConcurrent exercises create, close, and list operations // concurrently to verify EmbeddedDoltStore handles concurrent CLI invocations // without panics, data corruption, or deadlocks. diff --git a/cmd/bd/close_if_revision.go b/cmd/bd/close_if_revision.go index b0954d6f5..272a26c0e 100644 --- a/cmd/bd/close_if_revision.go +++ b/cmd/bd/close_if_revision.go @@ -8,8 +8,7 @@ import ( "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/debug" - "github.com/steveyegge/beads/internal/storage/uow" - "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/issueops" ) @@ -61,6 +60,10 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool Session: session, Force: force, ExpectedVersion: &expectedVersion, + // The library closes the molecule root this close completes, in the + // same transaction — and replays it on an idempotent re-close — where + // the store serves it (storage.ServesMoleculeAutoClose). + AutoCloseMolecule: storage.ServesMoleculeAutoClose(ctx, result.Store), }) if closeErr != nil { if reported, ok := reportIfRevisionFailure("closing", id, closeErr, &expectedVersion); ok { @@ -77,16 +80,11 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool fmt.Fprintf(os.Stderr, "warning: closing %s with %d open child issue(s) still active\n", result.ResolvedID, closeResult.OpenChildren) } - // Molecule auto-close is a retry-safe, fully state-derived post-close - // contract (close.go's autoCloseCompletedMolecule doc), so it must replay - // here exactly as it does on the batch route — on a real close AND on an - // idempotent re-close, since the guarded route bypasses BatchCloser - // entirely (mc-zndi7.75) and would otherwise leave a molecule's root - // stranded open forever. mutatedIDs := []string{result.ResolvedID} - if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, currentActor(), session); molID != "" { + if molID := reportMoleculeAutoClose(closeResult.AutoClosedMolecule, closeResult.MoleculeAutoCloseRefusal); molID != "" { mutatedIDs = append(mutatedIDs, molID) } + noticeUnservedMoleculeAutoClose(ctx, result.Store, result.ResolvedID, nil) if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "close", @@ -121,12 +119,13 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo } closeResult, closeErr := ops.Close(ctx, issueops.CloseRequest{ - Actor: currentActor(), - IssueID: id, - Reason: reason, - Session: session, - Force: force, - ExpectedVersion: &expectedVersion, + Actor: currentActor(), + IssueID: id, + Reason: reason, + Session: session, + Force: force, + ExpectedVersion: &expectedVersion, + AutoCloseMolecule: true, }) if closeErr != nil { if errors.Is(closeErr, context.Canceled) || errors.Is(closeErr, context.DeadlineExceeded) { @@ -146,48 +145,13 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo fmt.Fprintf(os.Stderr, "warning: closing %s with %d open child issue(s) still active\n", id, closeResult.OpenChildren) } - // Molecule auto-close is a retry-safe, fully state-derived post-close - // contract, so it must replay here too — on a real close AND on an - // idempotent re-close — exactly as closeProxiedRunPostClose drives it for - // the batch route (mc-zndi7.75). - runCloseIfRevisionProxiedPostClose(ctx, id, session) + reportMoleculeAutoClose(closeResult.AutoClosedMolecule, closeResult.MoleculeAutoCloseRefusal) SetLastTouchedID(id) reportClosedIfRevisionResult(id, reason, preCloseStatus, closeResult) return nil } -// runCloseIfRevisionProxiedPostClose re-drives molecule auto-close after a -// guarded proxied close, in its own unit of work exactly as -// closeProxiedRunPostClose runs it for the batch route: outside the close's -// own transaction, because it is outside that transaction's contract. A -// failure here is reported and swallowed — best effort, matching -// autoCloseCompletedMolecule's direct-route twin — so a molecule-healing -// hiccup never turns an otherwise-successful guarded close into a failure. -func runCloseIfRevisionProxiedPostClose(ctx context.Context, id, session string) { - if uowProvider == nil { - return - } - autoClosedMol, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*types.Issue, string, error) { - var warnings []string - mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, currentActor(), session, &warnings) - for _, w := range warnings { - fmt.Fprintf(os.Stderr, "Warning: %s\n", w) - } - if mol == nil { - return nil, "", nil - } - return mol, "bd: auto-close " + mol.ID, nil - }) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: post-close work failed: %v\n", err) - return - } - if autoClosedMol != nil && !jsonOutput { - debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(autoClosedMol.ID, autoClosedMol.Title)) - } -} - // reportClosedIfRevisionResult is the one-id success report shared by both // --if-revision close routes: the audit entry (suppressed for an idempotent // re-close, matching close.go's batch behavior) and the human/--json output. @@ -218,7 +182,7 @@ func reportClosedIfRevisionResult(id, reason, preCloseStatus string, closeResult } if jsonOutput { if closedIssue != nil { - _ = outputJSON([]*types.Issue{closedIssue}) + _ = outputJSON([]closedIssueJSON{newClosedIssueJSON(closedIssue, closeResult.AutoClosedMolecule, closeResult.MoleculeAutoCloseRefusal)}) } return } diff --git a/cmd/bd/close_if_revision_embedded_test.go b/cmd/bd/close_if_revision_embedded_test.go index 9b5721ed0..18867c6f1 100644 --- a/cmd/bd/close_if_revision_embedded_test.go +++ b/cmd/bd/close_if_revision_embedded_test.go @@ -48,9 +48,11 @@ func TestIfRevisionCloseReplaysMoleculeAutoClose(t *testing.T) { // Re-close the already-closed final step, this time GUARDED. The idempotent // guarded re-close must replay molecule auto-close and heal the - // stranded-open root exactly as the unguarded path does. + // stranded-open root exactly as the unguarded path does, and its --json + // report names the root it closed. rev := bdShowRevision(t, bd, dir, step2.ID) - bdRunOK(t, bd, dir, "close", step2.ID, "--if-revision", revStr(rev), "--reason", "retry") + out := bdClose(t, bd, dir, step2.ID, "--if-revision", revStr(rev), "--reason", "retry", "--json") + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{step2.ID}, root.ID) if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusClosed { t.Errorf("expected stranded-open molecule root %s healed by a guarded re-close of the final step, got %s", diff --git a/cmd/bd/close_if_revision_proxied_test.go b/cmd/bd/close_if_revision_proxied_test.go index 0657424ac..dfddc7dea 100644 --- a/cmd/bd/close_if_revision_proxied_test.go +++ b/cmd/bd/close_if_revision_proxied_test.go @@ -40,9 +40,11 @@ func TestProxiedServerIfRevisionCloseReplaysMoleculeAutoClose(t *testing.T) { // Re-close the already-closed final step, this time GUARDED. The // idempotent guarded re-close must replay molecule auto-close and heal - // the stranded-open root exactly as the unguarded path does. + // the stranded-open root exactly as the unguarded path does, and its + // --json report names the root it closed. rev := bdProxiedShowRevision(t, bd, p.dir, step2.ID) - bdProxiedClose(t, bd, p.dir, step2.ID, "--if-revision", proxiedRevStr(rev), "--reason", "retry") + out := bdProxiedClose(t, bd, p.dir, step2.ID, "--if-revision", proxiedRevStr(rev), "--reason", "retry", "--json") + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{step2.ID}, root.ID) if got := readStatus(t, db, root.ID); got != types.StatusClosed { t.Errorf("expected stranded-open molecule root %s healed by a guarded proxied re-close of the final step, got %q", diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 94baa1ad6..c82657266 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) func bdProxiedClose(t *testing.T, bd, dir string, args ...string) string { @@ -914,6 +915,31 @@ func TestProxiedServerClose2B(t *testing.T) { } }) + t.Run("json_reports_auto_closed_molecule", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "jacm") + root := bdProxiedCreate(t, bd, p.dir, "Molecule root", "-t", "epic", "--labels", "template") + s1 := bdProxiedCreate(t, bd, p.dir, "Step 1", "--parent", root.ID) + s2 := bdProxiedCreate(t, bd, p.dir, "Step 2", "--parent", root.ID) + s3 := bdProxiedCreate(t, bd, p.dir, "Step 3", "--parent", root.ID) + bdProxiedClose(t, bd, p.dir, s1.ID) + out := bdProxiedClose(t, bd, p.dir, "--json", s2.ID, s3.ID) + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{s2.ID, s3.ID}, root.ID) + }) + + t.Run("json_reports_molecule_auto_close_refusal", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "jmar") + root := bdProxiedCreate(t, bd, p.dir, "Molecule root", "-t", "epic", "--labels", "template", "--assignee", "someone-else") + s1 := bdProxiedCreate(t, bd, p.dir, "Step 1", "--parent", root.ID) + out := bdProxiedClose(t, bd, p.dir, "--actor", "json-closer", "--json", s1.ID) + requireCloseJSONReportsMoleculeRefusal(t, out, s1.ID, root.ID, + &issueops.CloseNotAssigneeError{IssueID: root.ID, Assignee: "someone-else", Actor: "json-closer"}) + if got := readStatus(t, openProxiedDB(t, p), root.ID); got != types.StatusOpen { + t.Errorf("a root assigned to another actor must stay open, got %q", got) + } + }) + t.Run("hooks_fire_on_close", func(t *testing.T) { t.Parallel() marker := filepath.Join(t.TempDir(), "on_close_marker") diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index cf94bba83..4e09ab602 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -5,13 +5,11 @@ import ( "errors" "fmt" "os" - "strings" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/domain" storeissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" @@ -65,14 +63,18 @@ type closeProxiedOutcome struct { closed bool auditOld string auditReason string + // autoClosedMol and autoCloseRefusal are what the library's molecule + // auto-close did inside the batch's transaction for this item. + autoClosedMol *types.Issue + autoCloseRefusal string } // closeProxiedPostClose is the work `bd close` does AFTER the closes have -// landed: molecule auto-close, --suggest-next and --continue. +// landed: --suggest-next and --continue. (The molecule auto-close is not here: +// the library runs it inside the batch's transaction.) type closeProxiedPostClose struct { unblocked []*types.Issue continueResult *ContinueResult - autoClosedMol *types.Issue warnings []string } @@ -146,6 +148,9 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin Session: in.session, Force: in.force, ClaimNext: closeClaimNextRequest(in.claimNext, in.continueOn), + // The library closes a molecule root its last step's close + // completes, inside this batch's transaction. + AutoCloseMolecule: true, }) if err != nil { return HandleErrorRespectJSON("%v", err) @@ -179,9 +184,17 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin claimedNextIssue = result.ClaimedNext.Issue } + for _, o := range outcomes { + if o.autoCloseRefusal != "" { + fmt.Fprintf(os.Stderr, "Warning: could not auto-close completed molecule: %s\n", o.autoCloseRefusal) + } + } + if !in.jsonOut { - if post.autoClosedMol != nil { - fmt.Printf("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(post.autoClosedMol.ID, post.autoClosedMol.Title)) + for _, o := range outcomes { + if o.autoClosedMol != nil { + fmt.Printf("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(o.autoClosedMol.ID, o.autoClosedMol.Title)) + } } if len(post.unblocked) > 0 { fmt.Printf("\nNewly unblocked:\n") @@ -200,9 +213,9 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin } if in.jsonOut && len(outcomes) > 0 { - closedIssues := make([]*types.Issue, len(outcomes)) + closedIssues := make([]closedIssueJSON, len(outcomes)) for i, o := range outcomes { - closedIssues[i] = o.after + closedIssues[i] = newClosedIssueJSON(o.after, o.autoClosedMol, o.autoCloseRefusal) } switch { case len(post.unblocked) > 0: @@ -352,12 +365,14 @@ func closeProxiedOutcomes(pre *closeProxiedPreflight, result issueops.CloseBatch after.Dependencies = nil } outcomes = append(outcomes, closeProxiedOutcome{ - id: item.IssueID, - before: before, - after: after, - closed: outcome.Changed, - auditOld: oldStatus, - auditReason: item.Reason, + id: item.IssueID, + before: before, + after: after, + closed: outcome.Changed, + auditOld: oldStatus, + auditReason: item.Reason, + autoClosedMol: outcome.AutoClosedMolecule, + autoCloseRefusal: outcome.MoleculeAutoCloseRefusal, }) reasons = append(reasons, item.Reason) } @@ -469,61 +484,38 @@ func closeProxiedRefusal(id string, err error) string { } } -// closeProxiedRunPostClose runs molecule auto-close, --suggest-next and -// --continue once the closes have committed. -// -// They are outside the batch's transaction because they are outside its -// contract, and outside is where the direct route has always run them: it -// calls autoCloseCompletedMolecule and AdvanceToNextStep after ops.Close -// returns, each in its own write. The visible consequence is a SECOND Dolt -// commit when a molecule actually auto-closes or --continue actually advances. -// A plain close writes nothing in this pass, names no commit message, and -// therefore still produces exactly one commit for the command. +// closeProxiedRunPostClose runs --suggest-next and --continue once the closes +// have committed. --suggest-next is a read in a unit of work of its own; +// --continue is the provider's MoleculeStepper role, which claims in its own +// transaction exactly as the direct route's does. func closeProxiedRunPostClose(ctx context.Context, args []string, in closeProxiedInput, outcomes []closeProxiedOutcome) closeProxiedPostClose { + var out closeProxiedPostClose if len(outcomes) == 0 { - return closeProxiedPostClose{} + return out } - post, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (closeProxiedPostClose, string, error) { - var out closeProxiedPostClose - var wrote []string - - for _, o := range outcomes { - mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, currentActor(), in.session, &out.warnings) - if mol != nil { - out.autoClosedMol = mol - wrote = append(wrote, "auto-close "+mol.ID) - } - } - - if in.suggestNext && len(args) == 1 { + if in.suggestNext && len(args) == 1 { + _, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (struct{}, error) { unblocked, warn := closeProxiedSuggestNext(ctx, uw, args[0]) out.unblocked = unblocked if warn != "" { out.warnings = append(out.warnings, warn) } + return struct{}{}, nil + }) + if err != nil { + out.warnings = append(out.warnings, fmt.Sprintf("post-close work failed: %v", err)) } - - if in.continueOn && len(args) == 1 { - cont, warn := closeProxiedContinue(ctx, uw, args[0], !in.noAuto) - out.continueResult = cont - if warn != "" { - out.warnings = append(out.warnings, warn) - } - if cont != nil && cont.AutoAdvanced && cont.NextStep != nil { - wrote = append(wrote, "advance to "+cont.NextStep.ID) - } - } - - if len(wrote) == 0 { - return out, "", nil - } - return out, "bd: " + strings.Join(wrote, "; "), nil - }) - if err != nil { - post.warnings = append(post.warnings, fmt.Sprintf("post-close work failed: %v", err)) } - return post + + if in.continueOn && len(args) == 1 { + cont, warn := closeProxiedContinue(ctx, args[0], !in.noAuto) + out.continueResult = cont + if warn != "" { + out.warnings = append(out.warnings, warn) + } + } + return out } func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID string) ([]*types.Issue, string) { @@ -534,52 +526,14 @@ func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID st return unblocked, "" } -func closeProxiedContinue(ctx context.Context, uw uow.UnitOfWork, closedID string, autoClaim bool) (*ContinueResult, string) { - result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, currentActor()) +func closeProxiedContinue(ctx context.Context, closedID string, autoClaim bool) (*ContinueResult, string) { + roles, err := proxiedMoleculeRoles() + if err != nil { + return nil, fmt.Sprintf("could not advance to next step: %v", err) + } + result, err := advanceMolecule(ctx, roles, closedID, autoClaim) if err != nil { return nil, fmt.Sprintf("could not advance to next step: %v", err) } return result, "" } - -func autoCloseProxiedCompletedMolecule(ctx context.Context, uw uow.UnitOfWork, closedStepID string, actorName, session string, warnings *[]string) *types.Issue { - moleculeID := proxiedFindParentMolecule(ctx, uw, closedStepID) - if moleculeID == "" { - return nil - } - - root, err := uw.IssueUseCase().GetIssue(ctx, moleculeID) - if err != nil || root == nil || root.Status == types.StatusClosed { - return nil - } - // A READ, and one that has to see this transaction. The auto-close decision - // is made from labels written earlier in the same unit of work, and - // issueops.Reader opens a transaction of its own, so it would answer from - // the last committed state instead. The follow-up is a reader role bound to - // a caller's transaction; until one exists this stays (ga-2ltro.12). - if labels, err := uw.LabelUseCase().GetLabels(ctx, moleculeID); err == nil { //nolint:forbidigo // in-transaction read; issueops.Reader would open its own - root.Labels = labels - } - if !shouldAutoCloseCompletedRoot(root) { - return nil - } - - progress, err := getMoleculeProgress(ctx, uowMolReader{uw: uw}, moleculeID) - if err != nil { - return nil - } - if progress.Completed < progress.Total { - return nil - } - - params := domain.CloseIssueParams{Reason: "all steps complete", Session: session} - if _, err := uw.IssueUseCase().CloseIssue(ctx, moleculeID, params, actorName); err != nil { - *warnings = append(*warnings, fmt.Sprintf("could not auto-close completed molecule %s: %v", moleculeID, err)) - return nil - } - return root -} - -func proxiedFindParentMolecule(ctx context.Context, uw uow.UnitOfWork, issueID string) string { - return findParentMolecule(ctx, uowMolReader{uw: uw}, issueID) -} diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 0a8fea444..64aeada9c 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -446,8 +446,11 @@ var createCmd = &cobra.Command{ defer func() { _ = parentLookupStore.Close() }() } - var inheritedLabels []string + noInheritLabels, _ := cmd.Flags().GetBool("no-inherit-labels") + inheritParentLabels := parentID != "" && !noInheritLabels if parentID != "" { + // A fail-fast with the message this command has always printed; + // the create role refuses a missing parent on its own as well. ctx := rootCtx _, err := parentLookupStore.GetIssue(ctx, parentID) if err != nil { @@ -456,16 +459,20 @@ var createCmd = &cobra.Command{ } return HandleError("failed to check parent issue: %v", err) } - - noInheritLabels, _ := cmd.Flags().GetBool("no-inherit-labels") - if !noInheritLabels { - inheritedLabels, _ = parentLookupStore.GetLabels(ctx, parentID) - } } - labels = mergeCreateLabels(labels, inheritedLabels) - if dryRun { + // The preview shows what the create role would inherit; the real + // create asks the role to inherit (CreateRequest.InheritLabelsFromParent) + // inside its own transaction. A failed read is reported, never + // previewed as "no labels". + if inheritParentLabels { + inherited, err := parentLookupStore.GetLabels(rootCtx, parentID) + if err != nil { + return HandleError("failed to read the labels of parent %s: %v", parentID, err) + } + labels = mergeCreateLabels(labels, inherited) + } return renderDryRun() } @@ -482,14 +489,12 @@ var createCmd = &cobra.Command{ } createCtx := rootCtx - if parentID != "" { - childID, err := store.GetNextChildID(rootCtx, parentID) - if err != nil { - return HandleError("%v", err) - } - explicitID = childID - createCtx = storage.WithReservedChildCounter(createCtx, parentID, childID) - } + // The --parent child id is minted by the Lifecycle role, on every + // backend and route: a create with a ParentID and no id of its own is + // ., minted inside the create's own transaction (conformance + // RunIssueOperationsCreateUnderAParentMintsTheNextChildID, served leg + // included), and the role stages the advanced counter itself. An + // explicit --id never reaches here with a parent (refused above). if explicitID != "" { _, err := validation.ValidateIDFormat(explicitID) @@ -568,21 +573,20 @@ var createCmd = &cobra.Command{ if err != nil { return HandleErrorRespectJSON("%v", err) } - // Label inheritance stays CLI-side (mergeCreateLabels above) because the - // dry-run preview needs it too; asking the facade to inherit as well - // would append the parent's labels a second time. + // The parent's labels are the role's to inherit, inside the create. if priorityDefault { issue.Priority = 0 } result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: currentActor(), - Issue: issue, - ParentID: parentID, - Dependencies: createDependencyRequests(depSpecs), - WaitsFor: waitsForRequest(waitsForSpec), - ForceIDPrefix: forceCreate, - IDPrefix: createIDPrefixOverride(), - DefaultPriority: priorityDefault, + Actor: currentActor(), + Issue: issue, + ParentID: parentID, + InheritLabelsFromParent: inheritParentLabels, + Dependencies: createDependencyRequests(depSpecs), + WaitsFor: waitsForRequest(waitsForSpec), + ForceIDPrefix: forceCreate, + IDPrefix: createIDPrefixOverride(), + DefaultPriority: priorityDefault, }) if err != nil { // RULING R1: an occupied --id is a refusal, not a silent full-row diff --git a/cmd/bd/critical_paths_test.go b/cmd/bd/critical_paths_test.go index 29c4d96f0..b6ff4fd2c 100644 --- a/cmd/bd/critical_paths_test.go +++ b/cmd/bd/critical_paths_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // TestBurnWisps tests direct wisp deletion via burnWisps. @@ -122,6 +124,23 @@ func TestBurnWisps(t *testing.T) { }) } +// closeWithMoleculeAutoClose (re-)closes a step through the store's Lifecycle +// role with AutoCloseMolecule set — the one path `bd close` takes on every +// route — so the library's in-transaction auto-close decides the root. The +// rule is state-derived, so re-closing an already-closed step drives it too. +func closeWithMoleculeAutoClose(t *testing.T, ctx context.Context, s storage.DoltStorage, stepID string) { + t.Helper() + ops, err := s.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := ops.Close(ctx, issueops.CloseRequest{ + Actor: "test-actor", IssueID: stepID, Reason: "done", Session: "test-session", AutoCloseMolecule: true, + }); err != nil { + t.Fatalf("close %s with molecule auto-close: %v", stepID, err) + } +} + // TestAutoCloseCompletedMolecule tests that closing the last step auto-closes the parent molecule. func TestAutoCloseCompletedMolecule(t *testing.T) { t.Parallel() @@ -179,7 +198,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Now trigger auto-close - autoCloseCompletedMolecule(ctx, s, step2.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, step2.ID) // Verify root is now closed updatedRoot, err := s.GetIssue(ctx, root.ID) @@ -235,7 +254,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Auto-close after closing step1 — step2 still open - autoCloseCompletedMolecule(ctx, s, step1.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, step1.ID) // Verify root is still open updatedRoot, err := s.GetIssue(ctx, root.ID) @@ -261,7 +280,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Should not panic or error — just no-op - autoCloseCompletedMolecule(ctx, s, orphan.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, orphan.ID) }) t.Run("NoOpForAlreadyClosedMolecule", func(t *testing.T) { @@ -297,7 +316,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Should not panic — early return because root is already closed - autoCloseCompletedMolecule(ctx, s, step.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, step.ID) }) t.Run("ClosesWhenReparentedStepExcluded", func(t *testing.T) { @@ -370,7 +389,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Trigger auto-close — should succeed since only 2 children remain - autoCloseCompletedMolecule(ctx, s, steps[1].ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, steps[1].ID) // Verify root auto-closed (reparented step should be excluded from molecule) updatedRoot, err := s.GetIssue(ctx, root.ID) diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 7fe321c17..ded0f415f 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -496,8 +496,10 @@ func validateFullGraphPlan(plan *GraphApplyPlan, cfg graphPlanConfig, opts Graph // duplicate ID (matching `bd create --id`), which for an atomic graph create // would silently rewrite the existing issue while reporting creation — a plan // must fail fast instead. Deliberately not overridable by --force: that flag -// vouches for a foreign prefix, not for destroying existing data. Best-effort -// by transport: exists is nil where the calling context has no store access. +// vouches for a foreign prefix, not for destroying existing data. The probe is +// exact on every backend, http included (its GetIssue answers a miss with +// storage.ErrNotFound like the local stores); exists is nil only where the +// command opened no store. func validateGraphApplyExplicitIDCollisions(plan *GraphApplyPlan, exists func(id string) (bool, error)) error { if exists == nil { return nil diff --git a/cmd/bd/mol_bond_compound_label_test.go b/cmd/bd/mol_bond_compound_label_test.go index 1bdc94b3c..30ce296b5 100644 --- a/cmd/bd/mol_bond_compound_label_test.go +++ b/cmd/bd/mol_bond_compound_label_test.go @@ -45,9 +45,6 @@ func (w *recordingMolWriter) DeleteIssue(context.Context, string, string) error func (w *recordingMolWriter) SetConfig(context.Context, string, string) error { panic("SetConfig: not part of the proto+proto bond") } -func (w *recordingMolWriter) ClaimStepIfOpen(context.Context, string, string) error { - panic("ClaimStepIfOpen: not part of the proto+proto bond") -} // TestBondProtoProtoLabelsTheCompoundAtCreation pins the property that makes // the compound's molecule label unable to land in the wrong plane: it travels diff --git a/cmd/bd/mol_current.go b/cmd/bd/mol_current.go index 6769f01ef..2fbfa9075 100644 --- a/cmd/bd/mol_current.go +++ b/cmd/bd/mol_current.go @@ -3,15 +3,16 @@ package main import ( "context" "fmt" - "sort" "strconv" "strings" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) // LargeMoleculeThreshold is the step count above which we show summary instead of full list. @@ -79,452 +80,100 @@ Use --limit or --range to view specific steps: if usesProxiedServer() { return runMolCurrentProxiedServer(rootCtx, args, agent, limit, rangeStr) } - - ctx := rootCtx - if store == nil { return HandleErrorRespectJSON("no database connection") } - - var rangeStart, rangeEnd int - if rangeStr != "" { - var err error - rangeStart, rangeEnd, err = parseRange(rangeStr) - if err != nil { - return HandleErrorRespectJSON("invalid range '%s': %v", rangeStr, err) - } + resolve := func(ctx context.Context, arg string) (string, error) { + return utils.ResolvePartialID(ctx, store, arg) } - - explicitSteps := limit > 0 || rangeStr != "" - - var molecules []*MoleculeProgress - - if len(args) == 1 { - moleculeID, err := utils.ResolvePartialID(ctx, store, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - - stats, err := store.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - - if stats.Total > LargeMoleculeThreshold && !explicitSteps && !jsonOutput { - printLargeMoleculeSummary(stats) - return nil - } - - progress, err := getMoleculeProgress(ctx, store, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - - if rangeStr != "" { - progress.Steps = filterStepsByRange(progress.Steps, rangeStart, rangeEnd) - } else if limit > 0 && len(progress.Steps) > limit { - progress.Steps = progress.Steps[:limit] - } - - molecules = append(molecules, progress) - } else { - molecules = findInProgressMolecules(ctx, store, agent) - - if len(molecules) == 0 { - molecules = findHookedMolecules(ctx, store, agent) - } - - if len(molecules) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Printf("No molecules in progress") - if agent != "" { - fmt.Printf(" for %s", agent) - } - fmt.Println(".") - fmt.Println("\nTo start work on a molecule:") - fmt.Println(" bd mol wisp create # Instantiate as ephemeral wisp") - fmt.Println(" bd update --claim # Claim a step") - return nil - } - } - - if jsonOutput { - return outputJSON(molecules) - } - - for i, mol := range molecules { - if i > 0 { - fmt.Println() - } - printMoleculeProgress(mol) - } - return nil + return runMolCurrent(rootCtx, store, resolve, args, agent, limit, rangeStr) }, } -// getMoleculeProgress loads a molecule and computes progress -func getMoleculeProgress(ctx context.Context, s molReader, moleculeID string) (*MoleculeProgress, error) { - subgraph, err := loadTemplateSubgraph(ctx, s, moleculeID) +// runMolCurrent is `bd mol current` on every route. The molecule's steps, +// their states, the progress counts and the current and next step are the +// library's (issueops.ViewMolecule, ActiveMoleculeIDs, ReadMoleculeProgress); +// what is here is the argument handling and the presentation. +func runMolCurrent(ctx context.Context, s storage.DoltStorage, resolve func(context.Context, string) (string, error), args []string, agent string, limit int, rangeStr string) error { + var rangeStart, rangeEnd int + if rangeStr != "" { + var err error + rangeStart, rangeEnd, err = parseRange(rangeStr) + if err != nil { + return HandleErrorRespectJSON("invalid range '%s': %v", rangeStr, err) + } + } + explicitSteps := limit > 0 || rangeStr != "" + + roles, err := currentMoleculeRoles(s) if err != nil { - return nil, err + return HandleErrorRespectJSON("%v", err) } - progress := &MoleculeProgress{ - MoleculeID: subgraph.Root.ID, - MoleculeTitle: subgraph.Root.Title, - Assignee: subgraph.Root.Assignee, - Total: len(subgraph.Issues) - 1, // Exclude root - } - - // Compute step readiness from within-molecule dependencies. - // Uses analyzeMoleculeParallel instead of GetReadyWork because GetReadyWork - // excludes ephemeral issues (wisp steps are ephemeral by definition). - // See: https://github.com/steveyegge/gastown/issues/1276 (historical reference) - analysis := analyzeMoleculeParallel(subgraph) - readyIDs := make(map[string]bool) - for id, info := range analysis.Steps { - if info.IsReady { - readyIDs[id] = true - } - } - - // Build step status list (exclude root) - var steps []*StepStatus - for _, issue := range subgraph.Issues { - if issue.ID == subgraph.Root.ID { - continue // Skip root - } - - step := &StepStatus{ - Issue: issue, - } - - switch issue.Status { - case types.StatusClosed: - step.Status = "done" - progress.Completed++ - case types.StatusInProgress: - step.Status = "current" - step.IsCurrent = true - progress.CurrentStep = issue - case types.StatusBlocked: - step.Status = "blocked" - default: - // Check if ready (unblocked) - if readyIDs[issue.ID] { - step.Status = "ready" - if progress.NextStep == nil { - progress.NextStep = issue - } - } else { - step.Status = "pending" - } - } - - steps = append(steps, step) - } - - // Sort steps by dependency order - sortStepsByDependencyOrder(steps, subgraph) - progress.Steps = steps - - // If no current step but there's a ready step, set it as next - if progress.CurrentStep == nil && progress.NextStep == nil { - for _, step := range steps { - if step.Status == "ready" { - progress.NextStep = step.Issue - break - } - } - } - - return progress, nil -} - -// findInProgressMolecules finds molecules with in_progress steps for an agent -func findInProgressMolecules(ctx context.Context, s molReader, agent string) []*MoleculeProgress { - var inProgressIssues []*types.Issue - - status := types.StatusInProgress - filter := types.IssueFilter{Status: &status} - if agent != "" { - filter.Assignee = &agent - } - allIssues, err := s.SearchIssues(ctx, "", filter) - if err == nil { - inProgressIssues = allIssues - } - - if len(inProgressIssues) == 0 { - return nil - } - - // Batch-find parent molecules for all in_progress issues (bd-hn4q) - issueIDs := make([]string, len(inProgressIssues)) - for i, issue := range inProgressIssues { - issueIDs[i] = issue.ID - } - moleculeRoots := findParentMolecules(ctx, s, issueIDs) - - moleculeMap := make(map[string]*MoleculeProgress) - for _, issue := range inProgressIssues { - moleculeID := moleculeRoots[issue.ID] - if moleculeID == "" { - continue - } - - if _, exists := moleculeMap[moleculeID]; !exists { - progress, err := getMoleculeProgress(ctx, s, moleculeID) - if err == nil { - moleculeMap[moleculeID] = progress - } - } - } - - // Convert to slice var molecules []*MoleculeProgress - for _, mol := range moleculeMap { - molecules = append(molecules, mol) - } - - // Sort by molecule ID for consistent output - sort.Slice(molecules, func(i, j int) bool { - return molecules[i].MoleculeID < molecules[j].MoleculeID - }) - - return molecules -} - -// findHookedMolecules finds molecules bonded to hooked issues for an agent. -// This is a fallback when no in_progress steps exist but a molecule is attached -// to the agent's hooked work via a "blocks" dependency. -func findHookedMolecules(ctx context.Context, s molReader, agent string) []*MoleculeProgress { - // Query for hooked issues assigned to the agent - status := types.StatusHooked - filter := types.IssueFilter{Status: &status} - if agent != "" { - filter.Assignee = &agent - } - hookedIssues, err := s.SearchIssues(ctx, "", filter) - if err != nil || len(hookedIssues) == 0 { - return nil - } - - // For each hooked issue, check if it IS a molecule or has blocks deps on one - moleculeMap := make(map[string]*MoleculeProgress) - for _, issue := range hookedIssues { - // Check if the hooked issue itself is a molecule (e.g., patrol wisps - // are directly hooked without a separate handoff bead). hq-3paz0m - if issue.IssueType == types.TypeEpic { - if _, exists := moleculeMap[issue.ID]; !exists { - progress, err := getMoleculeProgress(ctx, s, issue.ID) - if err == nil { - moleculeMap[issue.ID] = progress - continue - } - } - } - - deps, err := s.GetDependencyRecords(ctx, issue.ID) + if len(args) == 1 { + moleculeID, err := resolve(ctx, args[0]) if err != nil { - continue + return moleculeNotResolved(args[0], err) } - // Look for a blocks dependency pointing to a molecule (epic or template) - for _, dep := range deps { - if dep.Type != types.DepBlocks { - continue + if !explicitSteps && !jsonOutput { + stats, err := issueops.ReadMoleculeProgress(ctx, roles.molecule, moleculeID) + if err != nil { + return HandleErrorRespectJSON("loading molecule: %v", err) } - // The issue depends on (is blocked by) dep.DependsOnID - candidate, err := s.GetIssue(ctx, dep.DependsOnID) - if err != nil || candidate == nil { - continue - } - - // Check if candidate is a molecule (epic or has template label) - isMolecule := candidate.IssueType == types.TypeEpic - if !isMolecule { - for _, label := range candidate.Labels { - if label == BeadsTemplateLabel { - isMolecule = true - break - } - } - } - - if isMolecule { - if _, exists := moleculeMap[candidate.ID]; !exists { - progress, err := getMoleculeProgress(ctx, s, candidate.ID) - if err == nil { - moleculeMap[candidate.ID] = progress - } - } - } - } - } - - // Convert to slice - var molecules []*MoleculeProgress - for _, mol := range moleculeMap { - molecules = append(molecules, mol) - } - - // Sort by molecule ID for consistent output - sort.Slice(molecules, func(i, j int) bool { - return molecules[i].MoleculeID < molecules[j].MoleculeID - }) - - return molecules -} - -// findParentMolecules batch-finds the root molecule for multiple issue IDs. -// Returns a map of issueID → moleculeRootID for issues that belong to a molecule. -// Issues not part of a molecule are omitted from the result. -// -// This replaces the previous N+1 pattern where findParentMolecule was called -// in a loop, issuing GetDependencyRecords + GetIssue per level per issue. -// Instead, this walks parent-child chains level-by-level using batch queries, -// reducing O(N * depth) round-trips to O(depth). (bd-hn4q) -func findParentMolecules(ctx context.Context, s molReader, issueIDs []string) map[string]string { - if len(issueIDs) == 0 { - return nil - } - - // rootOf accumulates: startID -> rootID for chains that terminated - rootOf := make(map[string]string, len(issueIDs)) - - // current tracks: startID -> currentAncestorID (still walking up) - current := make(map[string]string, len(issueIDs)) - for _, id := range issueIDs { - current[id] = id - } - - // Walk up parent-child chains in batch, level by level - for depth := 0; depth < 50 && len(current) > 0; depth++ { - // Collect unique IDs at current level - seen := make(map[string]bool, len(current)) - toCheck := make([]string, 0, len(current)) - for _, curID := range current { - if !seen[curID] { - seen[curID] = true - toCheck = append(toCheck, curID) + if stats.Total > LargeMoleculeThreshold { + printLargeMoleculeSummary(stats) + return nil } } - // Batch fetch parent-child deps for all current ancestors - allDeps, err := s.GetDependencyRecordsForIssues(ctx, toCheck) + view, err := issueops.ViewMolecule(ctx, roles.molecule, moleculeID) if err != nil { + return HandleErrorRespectJSON("loading molecule: %v", err) + } + progress := moleculeProgressOf(view) + if rangeStr != "" { + progress.Steps = filterStepsByRange(progress.Steps, rangeStart, rangeEnd) + } else if limit > 0 && len(progress.Steps) > limit { + progress.Steps = progress.Steps[:limit] + } + molecules = append(molecules, progress) + } else { + ids, err := issueops.ActiveMoleculeIDs(ctx, roles.reader, roles.molecule, agent) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + molecules, err = viewMolecules(ctx, roles, ids) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + if len(molecules) == 0 { + if jsonOutput { + return outputJSON([]interface{}{}) + } + fmt.Printf("No molecules in progress") + if agent != "" { + fmt.Printf(" for %s", agent) + } + fmt.Println(".") + fmt.Println("\nTo start work on a molecule:") + fmt.Println(" bd mol wisp create # Instantiate as ephemeral wisp") + fmt.Println(" bd update --claim # Claim a step") return nil } + } - // Build parent lookup: childID -> parentID - parentOf := make(map[string]string, len(allDeps)) - for childID, deps := range allDeps { - for _, dep := range deps { - if dep.Type == types.DepParentChild { - parentOf[childID] = dep.DependsOnID - break - } - } + if jsonOutput { + return outputJSON(molecules) + } + for i, mol := range molecules { + if i > 0 { + fmt.Println() } - - // Advance chains that have parents, finalize those that don't - nextCurrent := make(map[string]string) - for startID, curID := range current { - if parent, ok := parentOf[curID]; ok { - nextCurrent[startID] = parent - } else { - rootOf[startID] = curID - } - } - current = nextCurrent + printMoleculeProgress(mol) } - - // Anything still walking after max depth — treat as root - for startID, curID := range current { - rootOf[startID] = curID - } - - // Batch fetch root issues to check if they're molecules - uniqueRoots := make(map[string]bool, len(rootOf)) - for _, rootID := range rootOf { - uniqueRoots[rootID] = true - } - rootIDs := make([]string, 0, len(uniqueRoots)) - for id := range uniqueRoots { - rootIDs = append(rootIDs, id) - } - - // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and - // the importIssueLookup TODO in import_shared.go for why this is not yet - // routed through the role. - rootIssues, err := s.GetIssuesByIDs(ctx, rootIDs) - if err != nil { - return nil - } - - isMolecule := make(map[string]bool, len(rootIssues)) - for _, issue := range rootIssues { - // `bd mol pour` creates roots with TypeMolecule (no template label), - // while distilled-from-epic templates use TypeEpic + BeadsTemplateLabel. - // Both shapes must be recognized so --continue can advance through - // poured molecules. See gastownhall/beads#3719. - if issue.IssueType == types.TypeEpic || issue.IssueType == types.TypeMolecule { - isMolecule[issue.ID] = true - continue - } - for _, label := range issue.Labels { - if label == BeadsTemplateLabel { - isMolecule[issue.ID] = true - break - } - } - } - - // Filter to only molecule roots - result := make(map[string]string, len(rootOf)) - for startID, rootID := range rootOf { - if isMolecule[rootID] { - result[startID] = rootID - } - } - - return result -} - -// findParentMolecule walks up the parent-child chain to find the root molecule -// for a single issue. Returns "" if the issue is not part of a molecule. -func findParentMolecule(ctx context.Context, s molReader, issueID string) string { - roots := findParentMolecules(ctx, s, []string{issueID}) - return roots[issueID] -} - -// sortStepsByDependencyOrder sorts steps by their dependency order -func sortStepsByDependencyOrder(steps []*StepStatus, subgraph *TemplateSubgraph) { - // Build dependency graph - depCount := make(map[string]int) // issue ID -> number of deps - for _, step := range steps { - depCount[step.Issue.ID] = 0 - } - - // Count blocking dependencies within the step set - stepIDs := make(map[string]bool) - for _, step := range steps { - stepIDs[step.Issue.ID] = true - } - - for _, dep := range subgraph.Dependencies { - if dep.Type == types.DepBlocks && stepIDs[dep.IssueID] && stepIDs[dep.DependsOnID] { - depCount[dep.IssueID]++ - } - } - - // Stable sort by dependency count (fewer deps first) - sort.SliceStable(steps, func(i, j int) bool { - return depCount[steps[i].Issue.ID] < depCount[steps[j].Issue.ID] - }) + return nil } // printMoleculeProgress prints the progress in human-readable format @@ -590,77 +239,6 @@ type ContinueResult struct { MoleculeID string `json:"molecule_id,omitempty"` } -// AdvanceToNextStep finds the next ready step in a molecule after closing a step. -// If autoClaim is true, it marks the next step as in_progress using optimistic -// concurrency control: the step's status is re-verified before claiming to -// guard against TOCTOU races where multiple agents identify and try to claim the -// same step concurrently. -// Returns nil if the issue is not part of a molecule. -func AdvanceToNextStep(ctx context.Context, s molWriter, closedStepID string, autoClaim bool, actorName string) (*ContinueResult, error) { - if s == nil { - return nil, fmt.Errorf("no database connection") - } - - // Get the closed step - closedStep, err := s.GetIssue(ctx, closedStepID) - if err != nil || closedStep == nil { - return nil, fmt.Errorf("could not get closed step: %w", err) - } - - result := &ContinueResult{ - ClosedStep: closedStep, - } - - // Find parent molecule - moleculeID := findParentMolecule(ctx, s, closedStepID) - if moleculeID == "" { - // Not part of a molecule - nothing to advance - return nil, nil - } - result.MoleculeID = moleculeID - - // Load molecule progress - progress, err := getMoleculeProgress(ctx, s, moleculeID) - if err != nil { - return nil, fmt.Errorf("could not load molecule: %w", err) - } - - // Check if molecule is complete - if progress.Completed >= progress.Total { - result.MolComplete = true - return result, nil - } - - // Collect all ready steps (not just the first) so we can fall back - // if a concurrent agent claims one before us. - var readySteps []*types.Issue - for _, step := range progress.Steps { - if step.Status == "ready" { - readySteps = append(readySteps, step.Issue) - } - } - - if len(readySteps) == 0 { - // No ready steps - might be blocked - return result, nil - } - - result.NextStep = readySteps[0] - - if autoClaim { - for _, candidate := range readySteps { - if err := s.ClaimStepIfOpen(ctx, candidate.ID, actorName); err == nil { - result.NextStep = candidate - result.AutoAdvanced = true - break - } - // This candidate was already claimed; try the next ready step - } - } - - return result, nil -} - // PrintContinueResult prints the result of advancing to the next step func PrintContinueResult(result *ContinueResult) { if result == nil { diff --git a/cmd/bd/mol_port.go b/cmd/bd/mol_port.go index e915f9030..68c146494 100644 --- a/cmd/bd/mol_port.go +++ b/cmd/bd/mol_port.go @@ -37,7 +37,6 @@ type molReader interface { GetConfig(ctx context.Context, key string) (string, error) IsInfraTypeCtx(ctx context.Context, t types.IssueType) bool GetCustomStatusesDetailed(ctx context.Context) ([]types.CustomStatus, error) - GetMoleculeProgress(ctx context.Context, moleculeID string) (*types.MoleculeProgressStats, error) GetMoleculeLastActivity(ctx context.Context, moleculeID string) (*types.MoleculeLastActivity, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) } @@ -58,7 +57,6 @@ type molWriter interface { CloseIssue(ctx context.Context, id, reason, actor string) error DeleteIssue(ctx context.Context, id, actor string) error SetConfig(ctx context.Context, key, value string) error - ClaimStepIfOpen(ctx context.Context, id, actor string) error } type storeMolWriter struct { @@ -103,26 +101,6 @@ func (w storeMolWriter) GetConfig(ctx context.Context, key string) (string, erro return w.DoltStorage.GetConfig(ctx, key) } -func (w storeMolWriter) ClaimStepIfOpen(ctx context.Context, id, actor string) error { - return w.DoltStorage.RunInTransaction(ctx, fmt.Sprintf("bd: advance to step %s", id), func(tx storage.Transaction) error { - current, err := tx.GetIssue(ctx, id) - if err != nil { - return err - } - if current == nil { - return fmt.Errorf("step %s not found", id) - } - if current.Status != types.StatusOpen { - return fmt.Errorf("step %s already claimed (status: %s)", id, current.Status) - } - return tx.UpdateIssue(ctx, id, map[string]interface{}{"status": types.StatusInProgress}, actor) - }) -} - -func newStandaloneStoreMolWriter(store storage.DoltStorage) storeMolWriter { - return storeMolWriter{DoltStorage: store} -} - type uowMolReader struct { uw uow.UnitOfWork } @@ -266,40 +244,6 @@ func (r uowMolReader) GetCustomStatusesDetailed(ctx context.Context) ([]types.Cu return r.uw.ConfigUseCase().GetCustomStatuses(ctx) } -func (r uowMolReader) GetMoleculeProgress(ctx context.Context, moleculeID string) (*types.MoleculeProgressStats, error) { - stats := &types.MoleculeProgressStats{MoleculeID: moleculeID} - - root, err := r.GetIssue(ctx, moleculeID) - if err != nil { - return nil, fmt.Errorf("failed to get molecule: %w", err) - } - if root != nil { - stats.MoleculeTitle = root.Title - } - - dependents, err := r.GetDependentsWithMetadata(ctx, moleculeID) - if err != nil { - return nil, fmt.Errorf("failed to get molecule children: %w", err) - } - - for _, dependent := range dependents { - if dependent.DependencyType != types.DepParentChild { - continue - } - stats.Total++ - switch dependent.Status { - case types.StatusClosed: - stats.Completed++ - case types.StatusInProgress: - stats.InProgress++ - if stats.CurrentStepID == "" { - stats.CurrentStepID = dependent.ID - } - } - } - return stats, nil -} - func (r uowMolReader) GetMoleculeLastActivity(ctx context.Context, moleculeID string) (*types.MoleculeLastActivity, error) { dependents, err := r.GetDependentsWithMetadata(ctx, moleculeID) if err != nil { @@ -459,16 +403,3 @@ func (w *uowMolWriter) DeleteIssue(ctx context.Context, id, actor string) error func (w *uowMolWriter) SetConfig(ctx context.Context, key, value string) error { return w.uw.ConfigUseCase().SetConfig(ctx, key, value) } - -func (w *uowMolWriter) ClaimStepIfOpen(ctx context.Context, id, actor string) error { - isWisp, err := w.isWisp(ctx, id) - if err != nil { - return err - } - if isWisp { - _, err := w.uw.IssueUseCase().ClaimWispIfOpen(ctx, id, actor) - return err - } - _, err = w.uw.IssueUseCase().ClaimIssueIfOpen(ctx, id, actor) - return err -} diff --git a/cmd/bd/mol_progress.go b/cmd/bd/mol_progress.go index df77f9359..531d7cd21 100644 --- a/cmd/bd/mol_progress.go +++ b/cmd/bd/mol_progress.go @@ -6,9 +6,11 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) var molProgressCmd = &cobra.Command{ @@ -43,102 +45,82 @@ Example: if usesProxiedServer() { return runMolProgressProxiedServer(rootCtx, args) } - - ctx := rootCtx - if store == nil { return HandleErrorRespectJSON("no database connection") } - - var moleculeID string - if len(args) == 1 { - resolved, err := utils.ResolvePartialID(ctx, store, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - moleculeID = resolved - } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, store, currentActor()) - if len(moleculeIDs) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Println("No molecules in progress.") - fmt.Println("\nUse: bd mol progress ") - return nil - } - moleculeID = moleculeIDs[0] + resolve := func(ctx context.Context, arg string) (string, error) { + return utils.ResolvePartialID(ctx, store, arg) } - - stats, err := store.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - if jsonOutput { - output := map[string]interface{}{ - "molecule_id": stats.MoleculeID, - "molecule_title": stats.MoleculeTitle, - "total": stats.Total, - "completed": stats.Completed, - "in_progress": stats.InProgress, - "current_step_id": stats.CurrentStepID, - } - if stats.Total > 0 { - output["percent"] = float64(stats.Completed) * 100 / float64(stats.Total) - } - if stats.FirstClosed != nil && stats.LastClosed != nil && stats.Completed > 1 { - duration := stats.LastClosed.Sub(*stats.FirstClosed) - if duration > 0 { - rate := float64(stats.Completed-1) / duration.Hours() - output["rate_per_hour"] = rate - remaining := stats.Total - stats.Completed - if rate > 0 { - etaHours := float64(remaining) / rate - output["eta_hours"] = etaHours - } - } - } - return outputJSON(output) - } - - printMoleculeProgressStats(stats) - return nil + return runMolProgress(rootCtx, store, resolve, args) }, } -// findInProgressMoleculeIDs finds molecule IDs with in_progress steps for an agent. -// This is a lightweight version that only returns IDs without loading subgraphs. -func findInProgressMoleculeIDs(ctx context.Context, s molReader, agent string) []string { - // Query for in_progress issues - status := types.StatusInProgress - filter := types.IssueFilter{Status: &status} - if agent != "" { - filter.Assignee = &agent - } - inProgressIssues, err := s.SearchIssues(ctx, "", filter) - if err != nil || len(inProgressIssues) == 0 { - return nil +// runMolProgress is `bd mol progress` on every route: the counts are +// issueops.ReadMoleculeProgress over the molecule's direct steps, and the +// molecule without an id is the first of issueops.InProgressMoleculeIDs. +func runMolProgress(ctx context.Context, s storage.DoltStorage, resolve func(context.Context, string) (string, error), args []string) error { + roles, err := currentMoleculeRoles(s) + if err != nil { + return HandleErrorRespectJSON("%v", err) } - // Batch-find parent molecules for all in_progress issues (bd-hn4q) - issueIDs := make([]string, len(inProgressIssues)) - for i, issue := range inProgressIssues { - issueIDs[i] = issue.ID - } - moleculeRoots := findParentMolecules(ctx, s, issueIDs) - - seen := make(map[string]bool) - var moleculeIDs []string - for _, issue := range inProgressIssues { - moleculeID := moleculeRoots[issue.ID] - if moleculeID != "" && !seen[moleculeID] { - seen[moleculeID] = true - moleculeIDs = append(moleculeIDs, moleculeID) + var moleculeID string + if len(args) == 1 { + resolved, err := resolve(ctx, args[0]) + if err != nil { + return moleculeNotResolved(args[0], err) } + moleculeID = resolved + } else { + moleculeIDs, err := issueops.InProgressMoleculeIDs(ctx, roles.reader, roles.molecule, currentActor()) + if err != nil { + return HandleErrorRespectJSON("finding molecules in progress: %v", err) + } + if len(moleculeIDs) == 0 { + if jsonOutput { + return outputJSON([]interface{}{}) + } + fmt.Println("No molecules in progress.") + fmt.Println("\nUse: bd mol progress ") + return nil + } + moleculeID = moleculeIDs[0] } - return moleculeIDs + stats, err := issueops.ReadMoleculeProgress(ctx, roles.molecule, moleculeID) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + + if jsonOutput { + output := map[string]interface{}{ + "molecule_id": stats.MoleculeID, + "molecule_title": stats.MoleculeTitle, + "total": stats.Total, + "completed": stats.Completed, + "in_progress": stats.InProgress, + "current_step_id": stats.CurrentStepID, + } + if stats.Total > 0 { + output["percent"] = float64(stats.Completed) * 100 / float64(stats.Total) + } + if stats.FirstClosed != nil && stats.LastClosed != nil && stats.Completed > 1 { + duration := stats.LastClosed.Sub(*stats.FirstClosed) + if duration > 0 { + rate := float64(stats.Completed-1) / duration.Hours() + output["rate_per_hour"] = rate + remaining := stats.Total - stats.Completed + if rate > 0 { + etaHours := float64(remaining) / rate + output["eta_hours"] = etaHours + } + } + } + return outputJSON(output) + } + + printMoleculeProgressStats(stats) + return nil } // printMoleculeProgressStats prints molecule progress in human-readable format diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index 04f16955e..7f7bd4dde 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -160,158 +160,55 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { } func runMolShowProxiedServer(ctx context.Context, arg string) error { - uw, err := proxiedOpenReadUOW(ctx) + moleculeID, err := proxiedResolvePartialID(ctx, arg) if err != nil { - return err - } - defer uw.Close(ctx) - - r := uowMolReader{uw: uw} - moleculeID, err := utils.ResolvePartialID(ctx, r, arg) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", arg) + return moleculeNotResolved(arg, err) } - subgraph, err := loadTemplateSubgraph(ctx, r, moleculeID) + roles, err := proxiedMoleculeRoles() if err != nil { return HandleErrorRespectJSON("loading molecule: %v", err) } - - if molShowParallel { - return showMoleculeWithParallel(subgraph) - } - return showMolecule(subgraph) + return runMolShow(ctx, roles, moleculeID) } func runMolCurrentProxiedServer(ctx context.Context, args []string, agent string, limit int, rangeStr string) error { - uw, err := proxiedOpenReadUOW(ctx) - if err != nil { - return err - } - defer uw.Close(ctx) - - r := uowMolReader{uw: uw} - - var rangeStart, rangeEnd int - if rangeStr != "" { - rangeStart, rangeEnd, err = parseRange(rangeStr) - if err != nil { - return HandleErrorRespectJSON("invalid range '%s': %v", rangeStr, err) - } - } - explicitSteps := limit > 0 || rangeStr != "" - - var molecules []*MoleculeProgress - - if len(args) == 1 { - moleculeID, err := utils.ResolvePartialID(ctx, r, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - - stats, err := r.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - if stats.Total > LargeMoleculeThreshold && !explicitSteps && !jsonOutput { - printLargeMoleculeSummary(stats) - return nil - } - - progress, err := getMoleculeProgress(ctx, r, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - if rangeStr != "" { - progress.Steps = filterStepsByRange(progress.Steps, rangeStart, rangeEnd) - } else if limit > 0 && len(progress.Steps) > limit { - progress.Steps = progress.Steps[:limit] - } - molecules = append(molecules, progress) - } else { - molecules = findInProgressMolecules(ctx, r, agent) - if len(molecules) == 0 { - molecules = findHookedMolecules(ctx, r, agent) - } - if len(molecules) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Printf("No molecules in progress") - if agent != "" { - fmt.Printf(" for %s", agent) - } - fmt.Println(".") - fmt.Println("\nTo start work on a molecule:") - fmt.Println(" bd mol wisp create # Instantiate as ephemeral wisp") - fmt.Println(" bd update --claim # Claim a step") - return nil - } - } - - if jsonOutput { - return outputJSON(molecules) - } - for i, mol := range molecules { - if i > 0 { - fmt.Println() - } - printMoleculeProgress(mol) - } - return nil + return runMolCurrent(ctx, nil, proxiedResolvePartialID, args, agent, limit, rangeStr) } func runMolProgressProxiedServer(ctx context.Context, args []string) error { + return runMolProgress(ctx, nil, proxiedResolvePartialID, args) +} + +// proxiedResolvePartialID resolves a partial id inside a read unit of work of +// its own. A unit of work that cannot be opened is reported as itself +// (proxiedOpenReadUOW) and answered as a reportedResolveError, so the caller +// does not report a connection failure as a molecule that was not found. +func proxiedResolvePartialID(ctx context.Context, arg string) (string, error) { uw, err := proxiedOpenReadUOW(ctx) if err != nil { - return err + return "", reportedResolveError{err: err} } defer uw.Close(ctx) + return utils.ResolvePartialID(ctx, uowMolReader{uw: uw}, arg) +} - r := uowMolReader{uw: uw} +// reportedResolveError is an id resolution that failed before it could look +// the id up, and whose failure was already reported. +type reportedResolveError struct{ err error } - var moleculeID string - if len(args) == 1 { - resolved, err := utils.ResolvePartialID(ctx, r, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - moleculeID = resolved - } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, r, currentActor()) - if len(moleculeIDs) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Println("No molecules in progress.") - fmt.Println("\nUse: bd mol progress ") - return nil - } - moleculeID = moleculeIDs[0] +func (e reportedResolveError) Error() string { return e.err.Error() } +func (e reportedResolveError) Unwrap() error { return e.err } + +// moleculeNotResolved reports a molecule id argument that did not resolve: +// "not found" when the lookup ran, and the failure as already reported when +// the lookup never ran (reportedResolveError). +func moleculeNotResolved(arg string, err error) error { + var reported reportedResolveError + if errors.As(err, &reported) { + return reported.err } - - stats, err := r.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - if jsonOutput { - output := map[string]interface{}{ - "molecule_id": stats.MoleculeID, - "molecule_title": stats.MoleculeTitle, - "total": stats.Total, - "completed": stats.Completed, - "in_progress": stats.InProgress, - "current_step_id": stats.CurrentStepID, - } - if stats.Total > 0 { - output["percent"] = float64(stats.Completed) * 100 / float64(stats.Total) - } - return outputJSON(output) - } - - printMoleculeProgressStats(stats) - return nil + return HandleErrorRespectJSON("molecule '%s' not found", arg) } func runMolLastActivityProxiedServer(ctx context.Context, arg string) error { diff --git a/cmd/bd/mol_ready_gated.go b/cmd/bd/mol_ready_gated.go index 749a7c7ac..ed256f380 100644 --- a/cmd/bd/mol_ready_gated.go +++ b/cmd/bd/mol_ready_gated.go @@ -9,6 +9,7 @@ import ( "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) // GatedMolecule represents a molecule ready for gate-resume dispatch @@ -197,7 +198,10 @@ func findGateReadyMolecules(ctx context.Context, s molReader) ([]*GatedMolecule, hookedIDs[i] = issue.ID hookedMolecules[issue.ID] = true // Mark hooked issue itself } - hookedRoots := findParentMolecules(ctx, s, hookedIDs) + hookedRoots, err := issueops.MoleculeRoots(ctx, rawMoleculeReader{s: s}.moleculeReader(), hookedIDs) + if err != nil { + return nil, fmt.Errorf("finding molecules of hooked issues: %w", err) + } for _, molID := range hookedRoots { hookedMolecules[molID] = true } @@ -226,7 +230,10 @@ func findGateReadyMolecules(ctx context.Context, s molReader) ([]*GatedMolecule, } // Batch-find molecule roots for all ready dependents - depMolRoots := findParentMolecules(ctx, s, readyDepIDs) + depMolRoots, err := issueops.MoleculeRoots(ctx, rawMoleculeReader{s: s}.moleculeReader(), readyDepIDs) + if err != nil { + return nil, fmt.Errorf("finding molecules of ready steps: %w", err) + } moleculeMap := make(map[string]*GatedMolecule) for _, gd := range readyDependents { diff --git a/cmd/bd/mol_roles.go b/cmd/bd/mol_roles.go new file mode 100644 index 000000000..2f4f76e4f --- /dev/null +++ b/cmd/bd/mol_roles.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// moleculeRoles is where `bd mol current`, `bd mol progress`, `bd ready --mol` +// and `bd close --continue` get their answers on the route this command runs +// on. Every field is a library role; the molecule rules (membership, readiness, +// progress, the advance and its claim) are the issueops functions composed over +// them, so the direct, proxied-server and remote routes answer one way. What +// stays in cmd/bd is presentation. +type moleculeRoles struct { + reader issueops.Reader + molecule issueops.MoleculeReader + stepper func() (issueops.MoleculeStepper, error) +} + +// directMoleculeRoles binds the roles to the opened store: a local backend, or +// a registered remote one (http), each through its own accessors. +func directMoleculeRoles(s storage.DoltStorage) (moleculeRoles, error) { + if s == nil { + return moleculeRoles{}, errors.New("no database connection") + } + reader, err := s.IssueReader() + if err != nil { + return moleculeRoles{}, err + } + molecule, err := issueops.NewMoleculeReader(s) + if err != nil { + return moleculeRoles{}, err + } + return moleculeRoles{reader: reader, molecule: molecule, stepper: s.MoleculeStepper}, nil +} + +// proxiedMoleculeRoles binds the roles to the proxied server's provider, +// through the provider's own accessors (where each of its layers is added). +func proxiedMoleculeRoles() (moleculeRoles, error) { + if uowProvider == nil { + return moleculeRoles{}, errors.New("proxied-server UOW provider not initialized") + } + reader, err := proxiedIssueReader() + if err != nil { + return moleculeRoles{}, err + } + src, ok := uowProvider.(interface { + BatchGetter() (issueops.BatchGetter, error) + IssueRelations() (issueops.Relations, error) + EdgeReader() (issueops.EdgeReader, error) + }) + if !ok { + return moleculeRoles{}, fmt.Errorf("proxied-server provider %T does not offer the molecule read roles", uowProvider) + } + molecule, err := issueops.NewMoleculeReader(src) + if err != nil { + return moleculeRoles{}, err + } + stepper := func() (issueops.MoleculeStepper, error) { + src, ok := uowProvider.(uow.MoleculeStepperSource) + if !ok { + return nil, fmt.Errorf("proxied-server provider %T does not offer the molecule advance", uowProvider) + } + return src.MoleculeStepper() + } + return moleculeRoles{reader: reader, molecule: molecule, stepper: stepper}, nil +} + +// currentMoleculeRoles picks the route. +func currentMoleculeRoles(s storage.DoltStorage) (moleculeRoles, error) { + if usesProxiedServer() { + return proxiedMoleculeRoles() + } + return directMoleculeRoles(s) +} + +// moleculeProgressOf presents a library MoleculeView as `bd mol current`'s +// MoleculeProgress. +func moleculeProgressOf(view *issueops.MoleculeView) *MoleculeProgress { + root := view.Graph.Root + progress := &MoleculeProgress{ + MoleculeID: root.ID, + MoleculeTitle: root.Title, + Assignee: root.Assignee, + CurrentStep: view.CurrentStep, + NextStep: view.NextStep, + Completed: view.Completed, + Total: view.Total, + } + for _, step := range view.Steps { + progress.Steps = append(progress.Steps, &StepStatus{Issue: step.Issue, Status: step.State, IsCurrent: step.IsCurrent}) + } + return progress +} + +// viewMolecules loads and presents each molecule, sorted by id. +func viewMolecules(ctx context.Context, roles moleculeRoles, ids []string) ([]*MoleculeProgress, error) { + out := make([]*MoleculeProgress, 0, len(ids)) + for _, id := range ids { + view, err := issueops.ViewMolecule(ctx, roles.molecule, id) + if err != nil { + return nil, fmt.Errorf("loading molecule %s: %w", id, err) + } + out = append(out, moleculeProgressOf(view)) + } + sort.Slice(out, func(i, j int) bool { return out[i].MoleculeID < out[j].MoleculeID }) + return out, nil +} + +// continueResultOf presents a library AdvanceResult as `bd close --continue`'s +// ContinueResult. A step in no molecule presents as nil, as it always has. +func continueResultOf(result issueops.AdvanceResult) *ContinueResult { + if result.MoleculeID == "" { + return nil + } + return &ContinueResult{ + ClosedStep: result.ClosedStep, + NextStep: result.NextStep, + AutoAdvanced: result.Claimed, + MolComplete: result.Complete, + MoleculeID: result.MoleculeID, + } +} + +// advanceMolecule runs `bd close --continue` through the route's +// MoleculeStepper role. +func advanceMolecule(ctx context.Context, roles moleculeRoles, closedStepID string, autoClaim bool) (*ContinueResult, error) { + stepper, err := roles.stepper() + if err != nil { + return nil, err + } + result, err := stepper.Advance(ctx, issueops.AdvanceRequest{Actor: currentActor(), ClosedStepID: closedStepID, AutoClaim: autoClaim}) + if err != nil { + return nil, err + } + return continueResultOf(result), nil +} + +// rawMoleculeReader binds the library's molecule rules to a molReader — the +// raw-method port the template commands (`mol ready --gated`, pour, bond) +// still read through, including the proxied route's in-transaction port. It +// is a data binding, like the in-transaction ones in internal/storage: it +// answers the three role reads from raw methods and decides nothing. +type rawMoleculeReader struct{ s molReader } + +// moleculeReader binds the library's rules to this port. +func (r rawMoleculeReader) moleculeReader() issueops.MoleculeReader { + return issueops.MoleculeReaderOver(r) +} + +func (r rawMoleculeReader) GetMany(ctx context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + issues, err := r.s.GetIssuesByIDs(ctx, req.IDs) + if err != nil { + return issueops.GetManyResult{}, err + } + found := make(map[string]*types.Issue, len(issues)) + for _, issue := range issues { + if issue != nil { + found[issue.ID] = issue + } + } + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + seen := map[string]bool{} + for _, id := range req.IDs { + if seen[id] { + continue + } + seen[id] = true + if issue := found[id]; issue != nil { + out.Issues = append(out.Issues, issue) + } else { + out.Missing = append(out.Missing, id) + } + } + return out, nil +} + +func (r rawMoleculeReader) Related(ctx context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + var ( + items []*types.IssueWithDependencyMetadata + err error + ) + if req.Direction == issueops.RelationIn { + items, err = r.s.GetDependentsWithMetadata(ctx, req.ID) + } else { + return nil, fmt.Errorf("%w: raw molecule reads answer inbound relations only", issueops.ErrValidation) + } + if err != nil { + return nil, err + } + out := make([]*issueops.RelatedIssue, 0, len(items)) + for _, item := range items { + if item != nil && (len(req.Types) == 0 || containsDependencyType(req.Types, item.DependencyType)) { + out = append(out, item) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (r rawMoleculeReader) ReadEdges(ctx context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + records, err := r.s.GetDependencyRecordsForIssues(ctx, req.IDs) + if err != nil { + return issueops.EdgeReadResult{}, err + } + out := issueops.EdgeReadResult{Anchors: make([]issueops.AnchorEdges, 0, len(req.IDs))} + for _, id := range req.IDs { + var edges []*issueops.Dependency + for _, edge := range records[id] { + if edge != nil && (len(req.Types) == 0 || containsDependencyType(req.Types, edge.Type)) { + edges = append(edges, edge) + } + } + out.Anchors = append(out.Anchors, issueops.AnchorEdges{ID: id, Edges: edges}) + } + return out, nil +} + +func containsDependencyType(set []types.DependencyType, t types.DependencyType) bool { + for _, candidate := range set { + if candidate == t { + return true + } + } + return false +} diff --git a/cmd/bd/mol_route_parity_integration_test.go b/cmd/bd/mol_route_parity_integration_test.go new file mode 100644 index 000000000..fea0b54f5 --- /dev/null +++ b/cmd/bd/mol_route_parity_integration_test.go @@ -0,0 +1,140 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +// TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute drives one molecule +// lifecycle — `bd create --parent`, `bd close --continue`, `bd ready --mol`, +// `bd mol current`, `bd mol progress` and the auto-close after the last step — +// through the real bd binary on the three routes a workspace can take: +// +// - direct: a server-mode workspace, the CLI on the opened store; +// - proxied: a --proxied-server workspace, the CLI on the provider; +// - remote: `bd connect` to a `bd serve` of a third server-mode workspace. +// +// Each step of the lifecycle has ONE library entry (issueops), so every route +// must leave the same rows behind and print the same JSON, timestamps aside. +// The fixture puts a ready step that ANOTHER actor holds first in line, which is +// the case the old direct route got wrong (a status-only claim that took it). +func TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute(t *testing.T) { + requireSharedProxiedServer(t) + bd := buildEmbeddedBD(t) + + directProject := newServerModeProject(t, bd, "mr") + proxiedProject := newSharedProxiedProject(t, bd, "mr") + servedProject := newServerModeProject(t, bd, "mr") + sp := startServe(t, bd, servedProject.dir, servedProject.env) + remoteDir := t.TempDir() + initGitRepoAt(t, remoteDir) + remoteEnv := bdEnv(remoteDir) + + routes := []moleculeFrontDoorRunner{ + {name: "direct", dir: directProject.dir, env: func(string) []string { return directProject.env }}, + {name: "proxied", dir: proxiedProject.dir, env: bdProxiedEnv}, + {name: "remote", dir: remoteDir, env: func(string) []string { return remoteEnv }}, + } + routes[2].mustRun(t, bd, "connect", "http://"+sp.addr, "--json") + + as := func(args ...string) []string { return append([]string{"--actor", "worker"}, args...) } + + // The fixture. Child ids are minted by the create role on every route. + for _, r := range routes { + r.mustRun(t, bd, as("create", "Route molecule", "--type", "molecule", "--id", "mr-root")...) + for i, title := range []string{"first step", "held step", "free step"} { + out := r.mustRun(t, bd, as("create", title, "--type", "task", "--parent", "mr-root", "--json")...) + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &created); err != nil { + t.Fatalf("[%s] parse create: %v\n%s", r.name, err, out) + } + if want := "mr-root." + string(rune('1'+i)); created.ID != want { + t.Fatalf("[%s] create --parent minted %q, want %q", r.name, created.ID, want) + } + } + r.mustRun(t, bd, as("update", "mr-root.1", "--status", "in_progress", "--assignee", "worker")...) + r.mustRun(t, bd, as("update", "mr-root.2", "--assignee", "someone-else")...) + } + + compare := func(t *testing.T, name string, args ...string) { + t.Helper() + var first any + for i, r := range routes { + got := dropMoleculeRouteRevisions(normalizedMoleculeJSON(t, r.mustRun(t, bd, as(args...)...))) + if i == 0 { + first = got + continue + } + if !reflect.DeepEqual(first, got) { + t.Errorf("%s diverged: %s answered\n%v\n%s answered\n%v", name, routes[0].name, first, r.name, got) + } + } + } + + compare(t, "mol current", "mol", "current", "mr-root", "--json") + compare(t, "mol progress", "mol", "progress", "mr-root", "--json") + compare(t, "ready --mol", "ready", "--mol", "mr-root", "--json") + + // close --continue: the next ready step is mr-root.2, but someone else + // holds it, so the advance claims mr-root.3 for worker and leaves .2 alone. + compare(t, "close --continue", "close", "mr-root.1", "--continue", "--json") + for _, r := range routes { + assertMoleculeRouteRow(t, bd, r, "mr-root.2", "open", "someone-else") + assertMoleculeRouteRow(t, bd, r, "mr-root.3", "in_progress", "worker") + assertMoleculeRouteRow(t, bd, r, "mr-root", "open", "") + } + + // Closing the last two steps closes the root, in the close's own + // transaction, on every route. + compare(t, "close free step", "close", "mr-root.3", "--json") + compare(t, "close held step", "close", "mr-root.2", "--force", "--json") + for _, r := range routes { + assertMoleculeRouteRow(t, bd, r, "mr-root", "closed", "") + out := r.mustRun(t, bd, "show", "mr-root", "--json") + if !strings.Contains(out, `"close_reason": "all steps complete"`) && !strings.Contains(out, `"close_reason":"all steps complete"`) { + t.Errorf("[%s] root close_reason is not the auto-close's:\n%s", r.name, out) + } + } +} + +func assertMoleculeRouteRow(t *testing.T, bd string, r moleculeFrontDoorRunner, id, status, assignee string) { + t.Helper() + out := r.mustRun(t, bd, "show", id, "--json") + var rows []struct { + Status string `json:"status"` + Assignee string `json:"assignee"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rows); err != nil || len(rows) != 1 { + t.Fatalf("[%s] parse show %s: %v\n%s", r.name, id, err, out) + } + if rows[0].Status != status || rows[0].Assignee != assignee { + t.Errorf("[%s] %s = %s/%q, want %s/%q", r.name, id, rows[0].Status, rows[0].Assignee, status, assignee) + } +} + +// dropMoleculeRouteRevisions removes the opaque row-version tokens, which each +// database mints on its own, and the lease's wall-clock fields. +func dropMoleculeRouteRevisions(value any) any { + switch v := value.(type) { + case []any: + for i := range v { + v[i] = dropMoleculeRouteRevisions(v[i]) + } + case map[string]any: + // The claim's lease clock is wall time, like the timestamps the shared + // normalizer already drops. + for _, key := range []string{"revision", "heartbeat_at", "lease_expires_at"} { + delete(v, key) + } + for key, item := range v { + v[key] = dropMoleculeRouteRevisions(item) + } + } + return value +} diff --git a/cmd/bd/mol_show.go b/cmd/bd/mol_show.go index 281b3d6b9..bc831c9af 100644 --- a/cmd/bd/mol_show.go +++ b/cmd/bd/mol_show.go @@ -1,8 +1,8 @@ package main import ( + "context" "fmt" - "sort" "strings" "github.com/spf13/cobra" @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) var ( @@ -21,6 +22,10 @@ var molShowCmd = &cobra.Command{ Short: "Show molecule details", Long: `Show molecule structure and details. +A molecule's steps are the issues linked to its root by parent-child +dependencies (bd create --parent, bd mol pour); an id that merely looks +hierarchical, like bd-xyz.4, does not make an issue a step. + The --parallel flag highlights parallelizable steps: - Steps with no blocking dependencies can run in parallel - Shows which steps are ready to start now @@ -54,18 +59,35 @@ Example: return HandleErrorRespectJSON("molecule '%s' not found", args[0]) } - subgraph, err := loadTemplateSubgraph(ctx, store, moleculeID) + roles, err := directMoleculeRoles(store) if err != nil { return HandleErrorRespectJSON("loading molecule: %v", err) } - - if molShowParallel { - return showMoleculeWithParallel(subgraph) - } - return showMolecule(subgraph) + return runMolShow(ctx, roles, moleculeID) }, } +// runMolShow loads the molecule through the library's one molecule read +// (issueops.LoadMolecule, over the route's roles) and renders it. Membership +// is the parent-child edge on every route, as for mol current, mol progress +// and ready --mol. +func runMolShow(ctx context.Context, roles moleculeRoles, moleculeID string) error { + graph, err := issueops.LoadMolecule(ctx, roles.molecule, moleculeID) + if err != nil { + return HandleErrorRespectJSON("loading molecule: %v", err) + } + subgraph := &MoleculeSubgraph{ + Root: graph.Root, + Issues: graph.Issues, + Dependencies: graph.Dependencies, + IssueMap: graph.IssueMap, + } + if molShowParallel { + return showMoleculeWithParallel(subgraph) + } + return showMolecule(subgraph) +} + func showMolecule(subgraph *MoleculeSubgraph) error { if jsonOutput { return outputJSON(map[string]interface{}{ @@ -150,272 +172,22 @@ func formatBondType(bondType string) string { } } -// ParallelInfo holds parallel analysis information for a step -type ParallelInfo struct { - StepID string `json:"step_id"` - Status string `json:"status"` - IsReady bool `json:"is_ready"` // Can start now (no blocking deps) - ParallelGroup string `json:"parallel_group"` // Group ID (steps with same group can parallelize) - BlockedBy []string `json:"blocked_by"` // IDs of open steps blocking this one - Blocks []string `json:"blocks"` // IDs of steps this one blocks - CanParallel []string `json:"can_parallel"` // IDs of steps that can run in parallel with this -} +// ParallelInfo is a step's readiness and parallelism, as the library's +// molecule analysis answers it. +type ParallelInfo = issueops.MoleculeStepInfo -// ParallelAnalysis holds the complete parallel analysis for a molecule -type ParallelAnalysis struct { - MoleculeID string `json:"molecule_id"` - TotalSteps int `json:"total_steps"` - ReadySteps int `json:"ready_steps"` - ParallelGroups map[string][]string `json:"parallel_groups"` // group ID -> step IDs - Steps map[string]*ParallelInfo `json:"steps"` -} +// ParallelAnalysis is the readiness and parallelism of a whole molecule. +type ParallelAnalysis = issueops.MoleculeAnalysis -// analyzeMoleculeParallel performs parallel detection on a molecule subgraph. -// Returns analysis of which steps can run in parallel. +// analyzeMoleculeParallel runs the library's molecule analysis +// (issueops.AnalyzeMolecule) over a loaded template subgraph. func analyzeMoleculeParallel(subgraph *MoleculeSubgraph) *ParallelAnalysis { - analysis := &ParallelAnalysis{ - MoleculeID: subgraph.Root.ID, - TotalSteps: len(subgraph.Issues), - ParallelGroups: make(map[string][]string), - Steps: make(map[string]*ParallelInfo), - } - - // Build dependency maps - // blockedBy[id] = set of issue IDs that block this issue - // blocks[id] = set of issue IDs that this issue blocks - blockedBy := make(map[string]map[string]bool) - blocks := make(map[string]map[string]bool) - parentChildren := make(map[string][]string) - - for _, issue := range subgraph.Issues { - blockedBy[issue.ID] = make(map[string]bool) - blocks[issue.ID] = make(map[string]bool) - } - - // Build child index for waits-for gate evaluation. - for _, dep := range subgraph.Dependencies { - if dep.Type == types.DepParentChild { - parentChildren[dep.DependsOnID] = append(parentChildren[dep.DependsOnID], dep.IssueID) - } - } - - // Process dependencies to find blocking relationships - for _, dep := range subgraph.Dependencies { - switch dep.Type { - case types.DepBlocks, types.DepConditionalBlocks: - // dep.IssueID depends on (is blocked by) dep.DependsOnID - if _, ok := blockedBy[dep.IssueID]; ok { - blockedBy[dep.IssueID][dep.DependsOnID] = true - } - if _, ok := blocks[dep.DependsOnID]; ok { - blocks[dep.DependsOnID][dep.IssueID] = true - } - case types.DepWaitsFor: - children := parentChildren[dep.DependsOnID] - if len(children) == 0 { - continue - } - - gate := types.ParseWaitsForGateMetadata(dep.Metadata) - if gate == types.WaitsForAnyChildren { - hasClosedChild := false - for _, childID := range children { - child := subgraph.IssueMap[childID] - if child != nil && child.Status == types.StatusClosed { - hasClosedChild = true - break - } - } - if hasClosedChild { - continue - } - } - - // For all-children (and unresolved any-children), each open child blocks the gate. - for _, childID := range children { - child := subgraph.IssueMap[childID] - if child == nil || child.Status == types.StatusClosed { - continue - } - - if _, ok := blockedBy[dep.IssueID]; ok { - blockedBy[dep.IssueID][childID] = true - } - if _, ok := blocks[childID]; ok { - blocks[childID][dep.IssueID] = true - } - } - } - } - - // Identify which steps are ready (no open blockers) - readySteps := make(map[string]bool) - for _, issue := range subgraph.Issues { - info := &ParallelInfo{ - StepID: issue.ID, - Status: string(issue.Status), - BlockedBy: []string{}, - Blocks: []string{}, - } - - // Check what blocks this step - for blockerID := range blockedBy[issue.ID] { - blocker := subgraph.IssueMap[blockerID] - if blocker != nil && blocker.Status != types.StatusClosed { - info.BlockedBy = append(info.BlockedBy, blockerID) - } - } - - // Check what this step blocks - for blockedID := range blocks[issue.ID] { - info.Blocks = append(info.Blocks, blockedID) - } - - // A step is ready if it's open/in_progress and has no open blockers - info.IsReady = (issue.Status == types.StatusOpen || issue.Status == types.StatusInProgress) && - len(info.BlockedBy) == 0 - - if info.IsReady { - readySteps[issue.ID] = true - analysis.ReadySteps++ - } - - // Sort for consistent output - sort.Strings(info.BlockedBy) - sort.Strings(info.Blocks) - - analysis.Steps[issue.ID] = info - } - - // Identify parallel groups: steps that can run concurrently - // Two steps can parallelize if: - // 1. Both are ready (or will be ready at same time) - // 2. Neither blocks the other (directly or transitively) - // 3. They share the same blocking depth (distance from root) - - // Calculate blocking depth for each step - depths := calculateBlockingDepths(subgraph, blockedBy) - - // Group steps by depth - steps at same depth can potentially parallelize - depthGroups := make(map[int][]string) - for id, depth := range depths { - depthGroups[depth] = append(depthGroups[depth], id) - } - - // For each depth level, identify parallel groups - groupCounter := 0 - for depth := 0; depth <= len(subgraph.Issues); depth++ { - stepsAtDepth := depthGroups[depth] - if len(stepsAtDepth) == 0 { - continue - } - - // Group steps that can parallelize (no blocking between them) - // Use union-find approach: start with each step in its own group - parent := make(map[string]string) - for _, id := range stepsAtDepth { - parent[id] = id - } - - find := func(x string) string { - for parent[x] != x { - parent[x] = parent[parent[x]] - x = parent[x] - } - return x - } - - union := func(x, y string) { - px, py := find(x), find(y) - if px != py { - parent[px] = py - } - } - - // Merge steps that CAN parallelize (no mutual blocking) - for i, id1 := range stepsAtDepth { - for j := i + 1; j < len(stepsAtDepth); j++ { - id2 := stepsAtDepth[j] - // Can parallelize if neither blocks the other - if !blocks[id1][id2] && !blocks[id2][id1] && - !blockedBy[id1][id2] && !blockedBy[id2][id1] { - union(id1, id2) - } - } - } - - // Collect groups - groups := make(map[string][]string) - for _, id := range stepsAtDepth { - root := find(id) - groups[root] = append(groups[root], id) - } - - // Assign group names and record can_parallel relationships - for _, members := range groups { - if len(members) > 1 { - groupCounter++ - groupName := fmt.Sprintf("group-%d", groupCounter) - analysis.ParallelGroups[groupName] = members - - // Update each step's parallel info - for _, id := range members { - info := analysis.Steps[id] - info.ParallelGroup = groupName - // Record all other members as can_parallel - for _, otherId := range members { - if otherId != id { - info.CanParallel = append(info.CanParallel, otherId) - } - } - sort.Strings(info.CanParallel) - } - } - } - } - - return analysis -} - -// calculateBlockingDepths calculates the "blocking depth" of each step. -// Depth 0 = no blockers, Depth 1 = blocked by depth-0 steps, etc. -func calculateBlockingDepths(subgraph *MoleculeSubgraph, blockedBy map[string]map[string]bool) map[string]int { - depths := make(map[string]int) - visited := make(map[string]bool) - - var calculateDepth func(id string) int - calculateDepth = func(id string) int { - if d, ok := depths[id]; ok { - return d - } - if visited[id] { - // Cycle detected, return 0 to break - return 0 - } - visited[id] = true - - maxBlockerDepth := -1 - for blockerID := range blockedBy[id] { - // Only count open blockers - blocker := subgraph.IssueMap[blockerID] - if blocker != nil && blocker.Status != types.StatusClosed { - blockerDepth := calculateDepth(blockerID) - if blockerDepth > maxBlockerDepth { - maxBlockerDepth = blockerDepth - } - } - } - - depth := maxBlockerDepth + 1 - depths[id] = depth - return depth - } - - for _, issue := range subgraph.Issues { - calculateDepth(issue.ID) - } - - return depths + return issueops.AnalyzeMolecule(&issueops.MoleculeGraph{ + Root: subgraph.Root, + Issues: subgraph.Issues, + Dependencies: subgraph.Dependencies, + IssueMap: subgraph.IssueMap, + }) } func showMoleculeWithParallel(subgraph *MoleculeSubgraph) error { diff --git a/cmd/bd/mol_show_proxied_integration_test.go b/cmd/bd/mol_show_proxied_integration_test.go index 58aaa699e..ad157f745 100644 --- a/cmd/bd/mol_show_proxied_integration_test.go +++ b/cmd/bd/mol_show_proxied_integration_test.go @@ -3,8 +3,14 @@ package main import ( + "context" "encoding/json" + "errors" + "io" + "strings" "testing" + + "github.com/steveyegge/beads/internal/storage/uow" ) func TestProxiedServerMolShow(t *testing.T) { @@ -96,3 +102,111 @@ func TestProxiedServerMolShow(t *testing.T) { } }) } + +// TestProxiedMolResolveReportsAnUnopenableReadAsItself pins that the proxied +// molecule resolver (proxiedResolvePartialID: mol show, current, progress and +// ready --mol) reports a read unit of work it cannot open as that failure, +// never as a molecule that was not found, while an id the lookup does not find +// still reads as not found (moleculeNotResolved). +func TestProxiedMolResolveReportsAnUnopenableReadAsItself(t *testing.T) { + oldProvider := uowProvider + t.Cleanup(func() { uowProvider = oldProvider }) + uowProvider = nil + ctx := context.Background() + const unopenable = "proxied-server UOW provider not initialized" + + var resolveErr error + stderr := captureStderr(t, func() { + _, resolveErr = proxiedResolvePartialID(ctx, "mol-x") + }) + var reported reportedResolveError + if !errors.As(resolveErr, &reported) { + t.Fatalf("proxiedResolvePartialID error = %v, want a reportedResolveError", resolveErr) + } + if !strings.Contains(stderr, unopenable) { + t.Errorf("proxiedResolvePartialID stderr = %q, want the open failure %q", stderr, unopenable) + } + + stderr = captureStderr(t, func() { + if got := moleculeNotResolved("mol-x", resolveErr); !errors.Is(got, reported.err) { + t.Errorf("moleculeNotResolved = %v, want the already-reported %v", got, reported.err) + } + }) + if stderr != "" { + t.Errorf("moleculeNotResolved reported an already-reported failure again: %q", stderr) + } + + stderr = captureStderr(t, func() { + _ = moleculeNotResolved("mol-x", errors.New("no issue found matching \"mol-x\"")) + }) + if !strings.Contains(stderr, "molecule 'mol-x' not found") { + t.Errorf("an id the lookup did not find reported %q, want molecule 'mol-x' not found", stderr) + } + + var showErr error + stderr = captureStderr(t, func() { showErr = runMolShowProxiedServer(ctx, "mol-x") }) + if showErr == nil || !strings.Contains(stderr, unopenable) || strings.Contains(stderr, "not found") { + t.Errorf("mol show over an unopenable read: err = %v, stderr = %q; want the open failure, not \"not found\"", showErr, stderr) + } +} + +// unopenableUOWProvider fails every NewUOW, as a proxied route's provider does +// when the server cannot be reached. +type unopenableUOWProvider struct{ err error } + +func (p unopenableUOWProvider) NewUOW(context.Context) (uow.UnitOfWork, error) { return nil, p.err } +func (unopenableUOWProvider) Close(context.Context) error { return nil } + +// TestProxiedReadyMolReportsAnUnopenableReadOnce pins `bd ready --mol` on the +// proxied route to the resolver contract above: a read unit of work that will +// not open is reported once, as itself, and is not followed by a "not found" +// for the molecule. Under --json that is one error document on stdout, not two. +func TestProxiedReadyMolReportsAnUnopenableReadOnce(t *testing.T) { + oldProvider, oldJSON := uowProvider, jsonOutput + t.Cleanup(func() { uowProvider, jsonOutput = oldProvider, oldJSON }) + const openFailure = "connection refused" + uowProvider = unopenableUOWProvider{err: errors.New(openFailure)} + ctx := context.Background() + + t.Run("text", func(t *testing.T) { + jsonOutput = false + var readyErr error + stderr := captureStderr(t, func() { + readyErr = runMoleculeReady(ctx, nil, proxiedResolvePartialID, "mol-x") + }) + if readyErr == nil { + t.Error("runMoleculeReady over an unopenable read returned nil") + } + if strings.Count(stderr, "Error:") != 1 || !strings.Contains(stderr, openFailure) || strings.Contains(stderr, "not found") { + t.Errorf("stderr = %q; want one Error: line reporting the open failure, not \"not found\"", stderr) + } + }) + + t.Run("json", func(t *testing.T) { + jsonOutput = true + var readyErr error + stdout := captureStdout(t, func() error { + readyErr = runMoleculeReady(ctx, nil, proxiedResolvePartialID, "mol-x") + return nil + }) + if readyErr == nil { + t.Error("runMoleculeReady over an unopenable read returned nil") + } + dec := json.NewDecoder(strings.NewReader(stdout)) + docs := 0 + for { + var doc map[string]any + err := dec.Decode(&doc) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("stdout is not a JSON document stream: %v\n%s", err, stdout) + } + docs++ + } + if docs != 1 || !strings.Contains(stdout, openFailure) || strings.Contains(stdout, "not found") { + t.Errorf("stdout carries %d JSON documents:\n%s\nwant one, reporting the open failure, not \"not found\"", docs, stdout) + } + }) +} diff --git a/cmd/bd/mol_test.go b/cmd/bd/mol_test.go index e24ffd10f..26819c645 100644 --- a/cmd/bd/mol_test.go +++ b/cmd/bd/mol_test.go @@ -9,7 +9,9 @@ import ( "testing" "github.com/steveyegge/beads/internal/formula" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) func TestParseDistillVar(t *testing.T) { @@ -1400,7 +1402,7 @@ func TestGetMoleculeProgress(t *testing.T) { } // Get progress - progress, err := getMoleculeProgress(ctx, s, root.ID) + progress, err := viewMoleculeProgress(ctx, s, root.ID) if err != nil { t.Fatalf("getMoleculeProgress failed: %v", err) } @@ -1486,19 +1488,19 @@ func TestFindParentMolecule(t *testing.T) { } // Find parent molecule from grandchild - moleculeID := findParentMolecule(ctx, s, grandchild.ID) + moleculeID := mustFindParentMolecule(t, ctx, s, grandchild.ID) if moleculeID != root.ID { t.Errorf("findParentMolecule(grandchild) = %q, want %q", moleculeID, root.ID) } // Find parent molecule from child - moleculeID = findParentMolecule(ctx, s, child.ID) + moleculeID = mustFindParentMolecule(t, ctx, s, child.ID) if moleculeID != root.ID { t.Errorf("findParentMolecule(child) = %q, want %q", moleculeID, root.ID) } // Find parent molecule from root - moleculeID = findParentMolecule(ctx, s, root.ID) + moleculeID = mustFindParentMolecule(t, ctx, s, root.ID) if moleculeID != root.ID { t.Errorf("findParentMolecule(root) = %q, want %q", moleculeID, root.ID) } @@ -1515,7 +1517,7 @@ func TestFindParentMolecule(t *testing.T) { } // Should return empty for orphan - moleculeID = findParentMolecule(ctx, s, orphan.ID) + moleculeID = mustFindParentMolecule(t, ctx, s, orphan.ID) if moleculeID != "" { t.Errorf("findParentMolecule(orphan) = %q, want empty", moleculeID) } @@ -1587,7 +1589,10 @@ func TestFindParentMoleculesBatch(t *testing.T) { } // Batch-find molecule roots for all issues at once - roots := findParentMolecules(ctx, s, []string{grandchild.ID, child.ID, root.ID, orphan.ID}) + roots, err := moleculeRootsOf(ctx, s, []string{grandchild.ID, child.ID, root.ID, orphan.ID}) + if err != nil { + t.Fatalf("findParentMolecules: %v", err) + } if got := roots[grandchild.ID]; got != root.ID { t.Errorf("findParentMolecules[grandchild] = %q, want %q", got, root.ID) @@ -1602,9 +1607,9 @@ func TestFindParentMoleculesBatch(t *testing.T) { t.Errorf("findParentMolecules[orphan] = %q, want empty", got) } - // Empty input should return nil - if result := findParentMolecules(ctx, s, nil); result != nil { - t.Errorf("findParentMolecules(nil) = %v, want nil", result) + // Empty input answers no roots. + if result, err := moleculeRootsOf(ctx, s, nil); err != nil || len(result) != 0 { + t.Errorf("MoleculeRoots(nil) = %v, %v, want none", result, err) } } @@ -1668,12 +1673,15 @@ func TestFindParentMolecule_RootShapes(t *testing.T) { if tt.isMolecule { wantSingle = root.ID } - if got := findParentMolecule(ctx, s, child.ID); got != wantSingle { + if got := mustFindParentMolecule(t, ctx, s, child.ID); got != wantSingle { t.Errorf("findParentMolecule(child) = %q, want %q", got, wantSingle) } wantBatch := map[string]string{child.ID: wantSingle} - gotBatch := findParentMolecules(ctx, s, []string{child.ID}) + gotBatch, err := moleculeRootsOf(ctx, s, []string{child.ID}) + if err != nil { + t.Fatalf("findParentMolecules: %v", err) + } if got := gotBatch[child.ID]; got != wantBatch[child.ID] { t.Errorf("findParentMolecules(child) = %q, want %q", got, wantBatch[child.ID]) } @@ -1733,7 +1741,7 @@ func TestAdvanceToNextStep_PouredMolecule(t *testing.T) { t.Fatalf("Failed to add blocking dep: %v", err) } - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "test") + result, err := advanceForTest(ctx, s, step1.ID, true, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -1815,7 +1823,7 @@ func TestFindHookedMolecules(t *testing.T) { } // Test: findHookedMolecules should find the molecule for this agent - molecules := findHookedMolecules(ctx, s, "test-agent") + molecules := mustFindHookedMolecules(t, ctx, s, "test-agent") if len(molecules) != 1 { t.Fatalf("findHookedMolecules() got %d molecules, want 1", len(molecules)) } @@ -1824,13 +1832,13 @@ func TestFindHookedMolecules(t *testing.T) { } // Test: different agent should not find the molecule - molecules = findHookedMolecules(ctx, s, "other-agent") + molecules = mustFindHookedMolecules(t, ctx, s, "other-agent") if len(molecules) != 0 { t.Errorf("findHookedMolecules(other-agent) got %d molecules, want 0", len(molecules)) } // Test: no agent filter should find the molecule - molecules = findHookedMolecules(ctx, s, "") + molecules = mustFindHookedMolecules(t, ctx, s, "") if len(molecules) != 1 { t.Errorf("findHookedMolecules('') got %d molecules, want 1", len(molecules)) } @@ -1890,7 +1898,7 @@ func TestAdvanceToNextStep(t *testing.T) { } // Advance from step1 (just closed) without auto-claim - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, false, "test") + result, err := advanceForTest(ctx, s, step1.ID, false, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -1916,7 +1924,7 @@ func TestAdvanceToNextStep(t *testing.T) { } // Now test with auto-claim - result, err = AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "test") + result, err = advanceForTest(ctx, s, step1.ID, true, "test") if err != nil { t.Fatalf("AdvanceToNextStep with auto-claim failed: %v", err) } @@ -1967,7 +1975,7 @@ func TestAdvanceToNextStepMoleculeComplete(t *testing.T) { } // Advance from the only step (molecule should be complete) - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, false, "test") + result, err := advanceForTest(ctx, s, step1.ID, false, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2000,7 +2008,7 @@ func TestAdvanceToNextStepOrphanIssue(t *testing.T) { } // Advance should return nil (not part of molecule) - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), orphan.ID, false, "test") + result, err := advanceForTest(ctx, s, orphan.ID, false, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2083,7 +2091,7 @@ func TestAdvanceToNextStepConcurrentClaim(t *testing.T) { // Now our agent tries to advance with autoClaim — step2 was identified as // ready during the read phase, but it's already in_progress. The function // should fall back to step3. - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "our-agent") + result, err := advanceForTest(ctx, s, step1.ID, true, "our-agent") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2169,7 +2177,7 @@ func TestAdvanceToNextStepAllClaimed(t *testing.T) { } // Advance with autoClaim — should fail gracefully (no steps to claim) - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "our-agent") + result, err := advanceForTest(ctx, s, step1.ID, true, "our-agent") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2846,43 +2854,6 @@ func TestAnalyzeMoleculeParallelMultipleArms(t *testing.T) { } } -// TestCalculateBlockingDepths tests the depth calculation -func TestCalculateBlockingDepths(t *testing.T) { - // Create chain: root -> step1 -> step2 -> step3 - root := &types.Issue{ID: "root", Status: types.StatusOpen} - step1 := &types.Issue{ID: "step1", Status: types.StatusOpen} - step2 := &types.Issue{ID: "step2", Status: types.StatusOpen} - step3 := &types.Issue{ID: "step3", Status: types.StatusOpen} - - subgraph := &MoleculeSubgraph{ - Root: root, - Issues: []*types.Issue{root, step1, step2, step3}, - IssueMap: map[string]*types.Issue{"root": root, "step1": step1, "step2": step2, "step3": step3}, - } - - blockedBy := map[string]map[string]bool{ - "root": {}, - "step1": {"root": true}, - "step2": {"step1": true}, - "step3": {"step2": true}, - } - - depths := calculateBlockingDepths(subgraph, blockedBy) - - if depths["root"] != 0 { - t.Errorf("root depth = %d, want 0", depths["root"]) - } - if depths["step1"] != 1 { - t.Errorf("step1 depth = %d, want 1", depths["step1"]) - } - if depths["step2"] != 2 { - t.Errorf("step2 depth = %d, want 2", depths["step2"]) - } - if depths["step3"] != 3 { - t.Errorf("step3 depth = %d, want 3", depths["step3"]) - } -} - // TestSpawnMoleculeEphemeralFlag verifies that spawnMolecule with ephemeral=true // creates issues with the Ephemeral flag set (bd-phin) func TestSpawnMoleculeEphemeralFlag(t *testing.T) { @@ -3370,3 +3341,70 @@ func TestPourRootNoVars(t *testing.T) { t.Errorf("Root description should be formula desc, got: %q", spawnedRoot.Description) } } + +// The molecule tests below drive the library's molecule rules through the +// store's own roles — the path every route takes — via these thin shims. + +func testMolRoles(t *testing.T, s storage.DoltStorage) moleculeRoles { + t.Helper() + roles, err := directMoleculeRoles(s) + if err != nil { + t.Fatalf("directMoleculeRoles: %v", err) + } + return roles +} + +func mustFindParentMolecule(t *testing.T, ctx context.Context, s storage.DoltStorage, id string) string { + t.Helper() + got, err := issueops.MoleculeRoot(ctx, testMolRoles(t, s).molecule, id) + if err != nil { + t.Fatalf("MoleculeRoot(%s): %v", id, err) + } + return got +} + +func moleculeRootsOf(ctx context.Context, s storage.DoltStorage, ids []string) (map[string]string, error) { + roles, err := directMoleculeRoles(s) + if err != nil { + return nil, err + } + return issueops.MoleculeRoots(ctx, roles.molecule, ids) +} + +func viewMoleculeProgress(ctx context.Context, s storage.DoltStorage, id string) (*MoleculeProgress, error) { + roles, err := directMoleculeRoles(s) + if err != nil { + return nil, err + } + view, err := issueops.ViewMolecule(ctx, roles.molecule, id) + if err != nil { + return nil, err + } + return moleculeProgressOf(view), nil +} + +func mustFindHookedMolecules(t *testing.T, ctx context.Context, s storage.DoltStorage, agent string) []*MoleculeProgress { + t.Helper() + roles := testMolRoles(t, s) + ids, err := issueops.ActiveMoleculeIDs(ctx, roles.reader, roles.molecule, agent) + if err != nil { + t.Fatalf("ActiveMoleculeIDs(%q): %v", agent, err) + } + got, err := viewMolecules(ctx, roles, ids) + if err != nil { + t.Fatalf("viewMolecules(%q): %v", agent, err) + } + return got +} + +func advanceForTest(ctx context.Context, s storage.DoltStorage, closedStepID string, autoClaim bool, actor string) (*ContinueResult, error) { + stepper, err := s.MoleculeStepper() + if err != nil { + return nil, err + } + result, err := stepper.Advance(ctx, issueops.AdvanceRequest{Actor: actor, ClosedStepID: closedStepID, AutoClaim: autoClaim}) + if err != nil { + return nil, err + } + return continueResultOf(result), nil +} diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 7e7e8aacf..2837b483e 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -97,7 +97,13 @@ This is useful for agents executing molecules to see which steps can run next.`, if claimReady { return HandleErrorRespectJSON("--claim cannot be combined with --mol") } - return runMoleculeReady(cmd, molID) + if store == nil { + return HandleErrorRespectJSON("no database connection") + } + resolve := func(ctx context.Context, arg string) (string, error) { + return utils.ResolvePartialID(ctx, store, arg) + } + return runMoleculeReady(rootCtx, store, resolve, molID) } explain, _ := cmd.Flags().GetBool("explain") @@ -637,37 +643,32 @@ func runReadyExplain(_ *cobra.Command) error { return nil } -func runMoleculeReady(_ *cobra.Command, molIDArg string) error { - ctx := rootCtx - - if store == nil { - return HandleErrorRespectJSON("no database connection") - } - - moleculeID, err := utils.ResolvePartialID(ctx, store, molIDArg) +// runMoleculeReady is `bd ready --mol` on every route. The molecule's nodes, +// their readiness, the ready list and parallel groups are the library's one +// derivation (issueops.ViewMolecule, shared with `mol current`); what is here +// is presentation. +func runMoleculeReady(ctx context.Context, s storage.DoltStorage, resolve func(context.Context, string) (string, error), molIDArg string) error { + moleculeID, err := resolve(ctx, molIDArg) if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", molIDArg) + return moleculeNotResolved(molIDArg, err) } - - subgraph, err := loadTemplateSubgraph(ctx, store, moleculeID) + roles, err := currentMoleculeRoles(s) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + view, err := issueops.ViewMolecule(ctx, roles.molecule, moleculeID) if err != nil { return HandleErrorRespectJSON("loading molecule: %v", err) } - - // Get parallel analysis to find ready steps - analysis := analyzeMoleculeParallel(subgraph) - - // Collect ready steps + subgraph, analysis := view.Graph, view.Analysis var readySteps []*MoleculeReadyStep - for _, issue := range subgraph.Issues { + for _, issue := range view.Ready { info := analysis.Steps[issue.ID] - if info != nil && info.IsReady { - readySteps = append(readySteps, &MoleculeReadyStep{ - Issue: issue, - ParallelInfo: info, - ParallelGroup: info.ParallelGroup, - }) - } + readySteps = append(readySteps, &MoleculeReadyStep{ + Issue: issue, + ParallelInfo: info, + ParallelGroup: info.ParallelGroup, + }) } if jsonOutput { diff --git a/cmd/bd/ready_proxied_server.go b/cmd/bd/ready_proxied_server.go index 30a815f32..5f1df9d2e 100644 --- a/cmd/bd/ready_proxied_server.go +++ b/cmd/bd/ready_proxied_server.go @@ -53,6 +53,12 @@ func runReadyProxiedServer(cmd *cobra.Command, ctx context.Context) error { // same sweep from its role (uow.readyClaimer.ClaimNext). uow.WakeExpiredDefersAdvisory(ctx, uowProvider) + // --mol reads through the provider's roles, each in a unit of work of its + // own, so it runs before this route opens the one it holds below. + if in.molID != "" { + return runMoleculeReady(ctx, nil, proxiedResolvePartialID, in.molID) + } + uw, err := uowProvider.NewUOW(ctx) if err != nil { return HandleErrorRespectJSON("open unit of work: %v", err) @@ -62,8 +68,6 @@ func runReadyProxiedServer(cmd *cobra.Command, ctx context.Context) error { switch { case in.gated: return runReadyProxiedGated(ctx, uw, in) - case in.molID != "": - return runReadyProxiedMolecule(ctx, uw, in) case in.explain: return runReadyProxiedExplain(ctx, uw, in) default: @@ -397,89 +401,6 @@ func runReadyProxiedExplain(ctx context.Context, uw uow.UnitOfWork, _ readyInput return nil } -func runReadyProxiedMolecule(ctx context.Context, uw uow.UnitOfWork, in readyInput) error { - moleculeID := in.molID - subgraph, err := loadTemplateSubgraph(ctx, uowMolReader{uw: uw}, moleculeID) - if err != nil { - return HandleError("loading molecule: %v", err) - } - - analysis := analyzeMoleculeParallel(subgraph) - - var readySteps []*MoleculeReadyStep - for _, issue := range subgraph.Issues { - info := analysis.Steps[issue.ID] - if info != nil && info.IsReady { - readySteps = append(readySteps, &MoleculeReadyStep{ - Issue: issue, - ParallelInfo: info, - ParallelGroup: info.ParallelGroup, - }) - } - } - - if in.jsonOut { - output := MoleculeReadyOutput{ - MoleculeID: moleculeID, - MoleculeTitle: subgraph.Root.Title, - TotalSteps: analysis.TotalSteps, - ReadySteps: len(readySteps), - Steps: readySteps, - ParallelGroups: analysis.ParallelGroups, - } - _ = outputJSON(output) - return nil - } - - fmt.Printf("\n%s Ready steps in molecule: %s\n", ui.RenderAccent("🧪"), subgraph.Root.Title) - fmt.Printf(" ID: %s\n", moleculeID) - fmt.Printf(" Total: %d steps, %d ready\n", analysis.TotalSteps, len(readySteps)) - if len(readySteps) == 0 { - fmt.Printf("\n%s No ready steps (all blocked or completed)\n\n", ui.RenderWarn("✨")) - return nil - } - if len(analysis.ParallelGroups) > 0 { - fmt.Printf("\n%s Parallel Groups:\n", ui.RenderPass("⚡")) - for groupName, members := range analysis.ParallelGroups { - readyInGroup := 0 - for _, id := range members { - if info := analysis.Steps[id]; info != nil && info.IsReady { - readyInGroup++ - } - } - if readyInGroup > 0 { - fmt.Printf(" %s: %d ready\n", groupName, readyInGroup) - } - } - } - fmt.Printf("\n%s Ready steps:\n\n", ui.RenderPass("📋")) - for i, step := range readySteps { - groupAnnotation := "" - if step.ParallelGroup != "" { - groupAnnotation = fmt.Sprintf(" [%s]", ui.RenderAccent(step.ParallelGroup)) - } - fmt.Printf("%d. [%s] [%s] %s: %s%s\n", i+1, - ui.RenderPriority(step.Issue.Priority), - ui.RenderType(string(step.Issue.IssueType)), - ui.RenderID(step.Issue.ID), - step.Issue.Title, - groupAnnotation) - if len(step.ParallelInfo.CanParallel) > 0 { - readyParallel := []string{} - for _, pID := range step.ParallelInfo.CanParallel { - if pInfo := analysis.Steps[pID]; pInfo != nil && pInfo.IsReady { - readyParallel = append(readyParallel, pID) - } - } - if len(readyParallel) > 0 { - fmt.Printf(" Can run with: %v\n", readyParallel) - } - } - } - fmt.Println() - return nil -} - func runReadyProxiedGated(ctx context.Context, uw uow.UnitOfWork, _ readyInput) error { molecules, err := findGateReadyMolecules(ctx, uowMolReader{uw: uw}) if err != nil { diff --git a/cmd/bd/template.go b/cmd/bd/template.go index e8ad2d576..422acc1cf 100644 --- a/cmd/bd/template.go +++ b/cmd/bd/template.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "regexp" "sort" @@ -16,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" + publicops "github.com/steveyegge/beads/issueops" ) // configReader is the minimal slice of storage.Storage that config-reading @@ -26,8 +28,10 @@ type configReader interface { GetConfig(ctx context.Context, key string) (string, error) } -// BeadsTemplateLabel is the label used to identify Beads-based templates -const BeadsTemplateLabel = "template" +// BeadsTemplateLabel is the label used to identify Beads-based templates. It +// is the label molecule membership and auto-close read +// (issueops.MoleculeTemplateLabel), so the two cannot drift apart. +const BeadsTemplateLabel = publicops.MoleculeTemplateLabel // variablePattern matches {{variable}} placeholders var variablePattern = regexp.MustCompile(`\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}`) @@ -212,8 +216,17 @@ func loadDescendants(ctx context.Context, s molReader, subgraph *TemplateSubgrap // Hierarchical IDs follow the pattern: parentID.N (e.g., "gt-abc.1", "gt-abc.2") hierarchicalChildren, err := findHierarchicalChildren(ctx, s, parentID) if err != nil { - // Non-fatal: continue with what we have - return nil + // The parent-child edges read above are the molecule's membership; + // this id-pattern scan only heals children whose edge is missing. A + // backend that cannot express an id-prefix search (a remote backend) + // therefore loads the edge-linked subgraph without the heal. Any other + // failure is a failed read and is returned, not treated as "no + // children". + var unsupported *storage.ErrUnsupported + if errors.As(err, &unsupported) { + return nil + } + return fmt.Errorf("failed to find hierarchical children of %s: %w", parentID, err) } for _, child := range hierarchicalChildren { diff --git a/docs/reference/json-schema.md b/docs/reference/json-schema.md index 8f2f19e7c..ae6aeb2ac 100644 --- a/docs/reference/json-schema.md +++ b/docs/reference/json-schema.md @@ -220,6 +220,29 @@ items, plus: `bd ready` also withholds because a blocked parent blocks its children, carries no `gated_by` +### bd close --json + +Returns a top-level JSON array with one entry per closed issue, in the order +the IDs were given; an issue that was already closed is reported too. Each +entry is the closed issue (`status` is `closed`, with `closed_at` and +`close_reason`) without its `dependencies`, plus two optional members present +only when that close completed a molecule (see +[Molecules](/workflows/molecules)): +- `auto_closed_molecule` (object, optional): The molecule root the close also + closed, as an issue object whose `close_reason` is `all steps complete` +- `molecule_auto_close_refusal` (string, optional): Why the molecule root + stayed open — for example an open blocker on the root, or a root assigned + to someone else. Human-readable prose; the step's close itself succeeded + +When one command closes several steps of the same molecule, only the first +of their entries carries these members. + +With `--suggest-next`, `--continue`, or `--claim-next`, the output is an +object instead: the array moves under `closed`, beside `unblocked` (issue +array), `continue` (`closed_step`, `next_step`, `auto_advanced`, +`molecule_complete`, `molecule_id`), or `claimed` (issue) respectively. When +the flag has nothing to report, the plain array is returned. + ### `import --json` Returns a summary object when `--json` is active: diff --git a/docs/workflows/molecules.md b/docs/workflows/molecules.md index 38c8e722e..01e10ad42 100644 --- a/docs/workflows/molecules.md +++ b/docs/workflows/molecules.md @@ -74,6 +74,11 @@ bd mol show --parallel # Highlight steps that can run concurrentl bd dep tree # Shows full hierarchy ``` +A molecule's steps are the issues linked to its root by a `parent-child` +dependency (what `--parent` and `bd mol pour` create). An ID that only looks +hierarchical, such as `bd-xyz.4` with no such dependency, does not make an +issue a step. + ## Working with Molecules ### The Execution Model @@ -146,8 +151,17 @@ bd close bd-xyz.1 --reason "Done" # Check what's ready next bd ready --mol bd-xyz + +# Or complete a step and claim the next ready one in one command +bd close bd-xyz.1 --continue ``` +`--continue` claims the next ready step for you (it becomes `in_progress` and +yours), skipping any step someone else already holds; add `--no-auto` to see +the next step without claiming it. Against a `bd serve` that cannot advance a +molecule, `bd close --continue` refuses before closing anything, so a step is +never left closed with the next one unclaimed. + ### Viewing Progress ```bash @@ -175,9 +189,46 @@ Completed Molecule Closed / Squashed / Burned ``` -Closing the last child does not close the molecule root — epics stay open as -close-eligible work until closed explicitly (`bd epic close-eligible` sweeps -them). For cleanup of the beads themselves: +### When the Last Step Closes + +Closing a molecule's last open step closes its root in the same command, with +the reason `all steps complete`: + +```bash +bd close bd-xyz.3 +# ✓ Auto-closed completed molecule bd-xyz — Release 1.0.0 +# ✓ Closed bd-xyz.3 — Verify: Closed +``` + +Not every root closes this way: + +| Root | After its last step closes | +|------|----------------------------| +| Poured molecule (`bd mol pour`) or wisp (`bd mol wisp`) | Closes | +| Epic with the `template` label | Closes | +| Any other epic | Stays open as close-eligible work: close it yourself, or sweep with `bd epic close-eligible` | + +A root that `bd close` would refuse on its own stays open too — for example, +it has an open blocker, it is pinned, or it is assigned to someone else. The +step still closes, and a warning names the root and the reason: + +```bash +bd close bd-abc.2 +# Warning: could not auto-close completed molecule: cannot close bd-abc: assignee is "bob", actor is "alice"; reclaim or use --force to override +# ✓ Closed bd-abc.2 — Check: Closed +``` + +Once the obstacle is gone, `bd close ` closes the root, and so does +closing the last step again. + +A `bd serve` that does not close molecule roots for its clients (an older +server, or one whose orchestrator manages molecules) leaves the root open: +`bd close` prints a note naming it, and `bd close ` closes it. + +With `--json`, the step's entry reports the closed root, or why it stayed +open — see the [JSON Output Schema Contract](/reference/json-schema). + +For cleanup of the beads themselves: - `bd mol squash ` condenses a molecule's ephemeral children into a permanent digest issue. diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 299076f2a..4d50cf5c1 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "httpapi", srcs = [ + "advance_molecule.go", "auth.go", "batch_apply.go", "batch_close.go", @@ -63,6 +64,7 @@ go_library( go_test( name = "httpapi_test", srcs = [ + "advance_molecule_test.go", "auth_test.go", "batch_apply_test.go", "batch_close_test.go", diff --git a/internal/httpapi/advance_molecule.go b/internal/httpapi/advance_molecule.go new file mode 100644 index 000000000..3679ca4aa --- /dev/null +++ b/internal/httpapi/advance_molecule.go @@ -0,0 +1,64 @@ +package httpapi + +import ( + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// advanceMoleculeRequestMembers is the document's member list for +// AdvanceMoleculeRequest, refused BY NAME for the reason every other body on +// this surface is. +var advanceMoleculeRequestMembers = []string{claimActorMember, "auto_claim"} + +// handleAdvanceMolecule moves a molecule on after one of its steps closed: the +// operation behind `bd close --continue`. Decode, issueops.MoleculeStepper, +// encode — the readiness rule and the claim are the role's. +func (s *Server) handleAdvanceMolecule(w http.ResponseWriter, r *http.Request) { + id := r.PathValue(customMethodIDValue) + if !s.requireNoQuery(w, r) { + return + } + if !s.requireJSONContent(w, r) { + return + } + members, res := decodeJSONObjectBody(w, r) + if res != nil { + s.fail(w, r, *res) + return + } + if offender, unknown := unknownMember(members, advanceMoleculeRequestMembers); unknown { + s.failUnknownMember(w, r, offender, advanceMoleculeRequestMembers) + return + } + actor, ok := s.bodyActor(w, r, members) + if !ok { + return + } + autoClaim, ok := s.booleanMember(w, r, members, "auto_claim") + if !ok { + return + } + + stepper, err := s.moleculeStepper(r) + if err != nil { + s.failErr(w, r, err) + return + } + result, err := stepper.Advance(r.Context(), issueops.AdvanceRequest{Actor: actor, ClosedStepID: id, AutoClaim: autoClaim}) + if err != nil { + s.failErr(w, r, err) + return + } + body := apigen.AdvanceMoleculeResponse{ + Complete: result.Complete, + Claimed: result.Claimed, + MoleculeId: optionalString(result.MoleculeID), + NextStep: result.NextStep, + } + if result.ClosedStep != nil { + body.ClosedStep = *result.ClosedStep + } + writeJSON(w, body) +} diff --git a/internal/httpapi/advance_molecule_test.go b/internal/httpapi/advance_molecule_test.go new file mode 100644 index 000000000..099234d74 --- /dev/null +++ b/internal/httpapi/advance_molecule_test.go @@ -0,0 +1,238 @@ +package httpapi + +import ( + "context" + "fmt" + "net/http" + "sort" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// handlerMolecule is a tiny stateful store behind the roles the advance is +// composed over (BatchGetter, Relations, EdgeReader, Claimer): root m (a +// molecule) <- m.1 (closed), m.2 (open). The handler tests below prove the +// wire edge; the rule itself is pinned by the molecule contract on every leg. +type handlerMolecule struct { + mu sync.Mutex + issues map[string]*types.Issue + deps []*types.Dependency + claims []issueops.ClaimRequest +} + +func newHandlerMolecule() *handlerMolecule { + h := &handlerMolecule{issues: map[string]*types.Issue{ + "m": {ID: "m", Title: "molecule", IssueType: types.TypeMolecule, Status: types.StatusOpen}, + "m.1": {ID: "m.1", Title: "one", IssueType: types.TypeTask, Status: types.StatusClosed}, + "m.2": {ID: "m.2", Title: "two", IssueType: types.TypeTask, Status: types.StatusOpen}, + "o": {ID: "o", Title: "orphan", IssueType: types.TypeTask, Status: types.StatusClosed}, + }} + h.deps = []*types.Dependency{ + {IssueID: "m.1", DependsOnID: "m", Type: types.DepParentChild}, + {IssueID: "m.2", DependsOnID: "m", Type: types.DepParentChild}, + } + return h +} + +func (h *handlerMolecule) GetMany(_ context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + for _, id := range req.IDs { + if issue := h.issues[id]; issue != nil { + clone := *issue + out.Issues = append(out.Issues, &clone) + } else { + out.Missing = append(out.Missing, id) + } + } + return out, nil +} + +func (h *handlerMolecule) Related(_ context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.issues[req.ID] == nil { + return nil, fmt.Errorf("%w: %s", issueops.ErrNotFound, req.ID) + } + var out []*issueops.RelatedIssue + for _, dep := range h.deps { + if dep.DependsOnID == req.ID { + out = append(out, &issueops.RelatedIssue{Issue: *h.issues[dep.IssueID], DependencyType: dep.Type}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (h *handlerMolecule) ReadEdges(_ context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + var out issueops.EdgeReadResult + for _, id := range req.IDs { + anchor := issueops.AnchorEdges{ID: id, Missing: h.issues[id] == nil} + for _, dep := range h.deps { + if dep.IssueID == id { + anchor.Edges = append(anchor.Edges, dep) + } + } + out.Anchors = append(out.Anchors, anchor) + } + return out, nil +} + +func (h *handlerMolecule) Claim(_ context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.claims = append(h.claims, req) + issue := h.issues[req.IssueID] + issue.Status, issue.Assignee = types.StatusInProgress, req.Actor + clone := *issue + return issueops.ClaimResult{Issue: &clone, Changed: true}, nil +} + +func newAdvanceServer(t *testing.T, h *handlerMolecule) *testServer { + t.Helper() + return newTestServer(t, rolesConfig(Config{BatchGetter: h, Relations: h, EdgeReader: h, Claimer: h})) +} + +func TestAdvanceMoleculeClaimsTheNextStepThroughTheRole(t *testing.T) { + h := newHandlerMolecule() + ts := newAdvanceServer(t, h) + + resp := ts.postBody(t, "/v0/beads/issues/m.1:advanceMolecule", "application/json", `{"actor":"alice","auto_claim":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["molecule_id"] != "m" || body["claimed"] != true || body["complete"] != false { + t.Fatalf("body = %v, want molecule m, claimed, incomplete", body) + } + next, _ := body["next_step"].(map[string]any) + if next["id"] != "m.2" || next["assignee"] != "alice" { + t.Errorf("next_step = %v, want m.2 assigned to alice", next) + } + closed, _ := body["closed_step"].(map[string]any) + if closed["id"] != "m.1" { + t.Errorf("closed_step = %v, want m.1", closed) + } + if len(h.claims) != 1 || h.claims[0] != (issueops.ClaimRequest{Actor: "alice", IssueID: "m.2"}) { + t.Errorf("the Claimer received %+v, want one claim of m.2 for alice", h.claims) + } +} + +func TestAdvanceMoleculeWithoutAutoClaimWritesNothing(t *testing.T) { + h := newHandlerMolecule() + ts := newAdvanceServer(t, h) + resp := ts.postBody(t, "/v0/beads/issues/m.1:advanceMolecule", "application/json", `{"actor":"alice"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["claimed"] != false || len(h.claims) != 0 { + t.Errorf("body = %v claims = %v, want nothing claimed", body, h.claims) + } +} + +func TestAdvanceMoleculeOutsideAMoleculeOmitsTheMolecule(t *testing.T) { + ts := newAdvanceServer(t, newHandlerMolecule()) + resp := ts.postBody(t, "/v0/beads/issues/o:advanceMolecule", "application/json", `{"actor":"alice","auto_claim":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if _, ok := body["molecule_id"]; ok { + t.Errorf("molecule_id = %v, want it absent for a step in no molecule", body["molecule_id"]) + } +} + +func TestAdvanceMoleculeRefusesBadRequestsBeforeTheRole(t *testing.T) { + h := newHandlerMolecule() + ts := newAdvanceServer(t, h) + for _, tc := range []struct { + path, body string + status int + }{ + {"/v0/beads/issues/m.1:advanceMolecule", `{"actor":"alice","extra":1}`, http.StatusBadRequest}, + {"/v0/beads/issues/m.1:advanceMolecule", `{"auto_claim":true}`, http.StatusBadRequest}, + {"/v0/beads/issues/m.1:advanceMolecule", `{"actor":"alice","auto_claim":"yes"}`, http.StatusBadRequest}, + {"/v0/beads/issues/nope:advanceMolecule", `{"actor":"alice"}`, http.StatusNotFound}, + } { + resp := ts.postBody(t, tc.path, "application/json", tc.body) + if resp.StatusCode != tc.status { + t.Errorf("%s %s: status = %d, want %d: %s", tc.path, tc.body, resp.StatusCode, tc.status, readAll(t, resp)) + } + } + if len(h.claims) != 0 { + t.Errorf("a refused request reached the Claimer: %+v", h.claims) + } +} + +func TestCloseForwardsAutoCloseMoleculeAndReportsTheRoot(t *testing.T) { + root := closedIssue("bd-root") + lifecycle := &roleLifecycle{closeResult: issueops.CloseResult{ + Issue: closedIssue("bd-1"), Changed: true, AutoClosedMolecule: root, + }} + ts := newCloseServer(t, lifecycle) + resp := ts.closeIssue(t, closePath, `{"actor":"alice","auto_close_molecule":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := lifecycle.closeRequests() + if len(got) != 1 || !got[0].AutoCloseMolecule { + t.Fatalf("the role received %+v, want AutoCloseMolecule set", got) + } + body := decodeBody(t, resp) + auto, _ := body["auto_closed_molecule"].(map[string]any) + if auto["id"] != "bd-root" { + t.Errorf("auto_closed_molecule = %v, want bd-root", body["auto_closed_molecule"]) + } + if _, ok := body["molecule_auto_close_refusal"]; ok { + t.Errorf("molecule_auto_close_refusal present on a clean auto-close: %v", body) + } + + lifecycle = &roleLifecycle{closeResult: issueops.CloseResult{ + Issue: closedIssue("bd-1"), Changed: true, MoleculeAutoCloseRefusal: "cannot close blocked issue", + }} + ts = newCloseServer(t, lifecycle) + resp = ts.closeIssue(t, closePath, `{"actor":"alice"}`) + if got := lifecycle.closeRequests(); len(got) != 1 || got[0].AutoCloseMolecule { + t.Fatalf("an absent member reached the role as %+v, want AutoCloseMolecule unset (opt-in)", got) + } + body = decodeBody(t, resp) + if body["molecule_auto_close_refusal"] != "cannot close blocked issue" { + t.Errorf("molecule_auto_close_refusal = %v, want the refusal", body["molecule_auto_close_refusal"]) + } + if _, ok := body["auto_closed_molecule"]; ok { + t.Errorf("auto_closed_molecule present with nothing auto-closed: %v", body) + } +} + +func TestBatchCloseForwardsAutoCloseMoleculeAndReportsItPerItem(t *testing.T) { + closer := &roleBatchCloser{result: issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{ + {IssueID: "bd-1", Issue: closedIssue("bd-1"), Changed: true}, + {IssueID: "bd-2", Issue: closedIssue("bd-2"), Changed: true, AutoClosedMolecule: closedIssue("bd-root")}, + }}} + ts := newBatchCloseServer(t, closer) + resp := ts.batchClose(t, `{"actor":"alice","items":[{"id":"bd-1"},{"id":"bd-2"}],"auto_close_molecule":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + closer.mu.Lock() + requests := append([]issueops.CloseBatchRequest(nil), closer.requests...) + closer.mu.Unlock() + if len(requests) != 1 || !requests[0].AutoCloseMolecule { + t.Fatalf("the role received %+v, want AutoCloseMolecule set", requests) + } + outcomes := outcomesOf(t, resp) + if _, ok := outcomes[0]["auto_closed_molecule"]; ok { + t.Errorf("outcome 0 = %v, want no auto-closed root", outcomes[0]) + } + auto, _ := outcomes[1]["auto_closed_molecule"].(map[string]any) + if auto["id"] != "bd-root" { + t.Errorf("outcome 1 auto_closed_molecule = %v, want bd-root", outcomes[1]["auto_closed_molecule"]) + } +} diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index afd6eedff..65380e5a8 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -321,6 +321,33 @@ type AddDependenciesResponse struct { Added []DependencyEdge `json:"added"` } +// AdvanceMoleculeRequest defines model for AdvanceMoleculeRequest. +type AdvanceMoleculeRequest struct { + // Actor Who advances, and with `auto_claim` the claimed step's assignee. `ClaimRequest.actor`'s rules exactly. + Actor string `json:"actor"` + + // AutoClaim Claim the next ready step for `actor`. Without it the operation writes nothing and only names the step. + AutoClaim *bool `json:"auto_claim,omitempty"` +} + +// AdvanceMoleculeResponse defines model for AdvanceMoleculeResponse. +type AdvanceMoleculeResponse struct { + // Claimed `next_step` was claimed for `actor`, and is the post-claim row. + Claimed bool `json:"claimed"` + + // ClosedStep A tracked work item. Property semantics documented here apply to every schema that repeats them below. + ClosedStep Issue `json:"closed_step"` + + // Complete Every step of the molecule is closed; `next_step` is then absent. + Complete bool `json:"complete"` + + // MoleculeId The step's molecule root. ABSENT when the step belongs to no molecule, and then nothing else below is set. + MoleculeId *string `json:"molecule_id,omitempty"` + + // NextStep A tracked work item. Property semantics documented here apply to every schema that repeats them below. + NextStep *Issue `json:"next_step,omitempty"` +} + // AnchorEdgeCount One anchor's edge cardinality, or the report that it is not there. type AnchorEdgeCount struct { // Count How many stored edges match, in the requested direction, after the type and status filters. Never negative. @@ -764,6 +791,11 @@ type BatchCloseRequest struct { // Actor Who is closing. `ClaimRequest.actor`'s rules exactly, and the value is recorded against every item. Actor string `json:"actor"` + // AutoCloseMolecule Behind the `issues.close.autoCloseMolecule` behavior token: an older server answers `unknown_parameter` for this name. + // + // Close each item's molecule root when the batch completed it: `CloseIssueRequest.auto_close_molecule` for every item that closed or was already closed, run after all the closes inside the same transaction. A batch that closes several steps of one molecule decides its root once, and the root it closed, or its refusal, is reported on the outcome of the first successful item under that root. + AutoCloseMolecule *bool `json:"auto_close_molecule,omitempty"` + // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard for EVERY item, and nothing else. It never bypasses the template guard or validation, and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. Force *bool `json:"force,omitempty"` @@ -920,6 +952,13 @@ type CloseIssueRequest struct { // Actor Who is closing the issue. `ClaimRequest.actor`'s rules exactly: the server trims it, then refuses an empty result, anything longer than 256 BYTES (the `maxLength` above counts characters — the byte limit is the binding one), and any control character including newline. The value reaches stored columns, event-stream attribution and the storage commit message, so an unvalidated newline would forge audit-trail lines. Actor string `json:"actor"` + // AutoCloseMolecule Behind the `issues.close.autoCloseMolecule` behavior token: an older server answers `unknown_parameter` for this name, and a client checks the token before sending it. + // + // Close the issue's molecule root IN THE SAME TRANSACTION when this close completed it: the root is an auto-closing molecule root (a `molecule`-typed or ephemeral root, or a template-labeled epic), it is open, and every step under it is now closed. The root close is unforced, guarded on the root revision read inside that transaction, and records `all steps complete` and this request's `session`. It runs for an idempotent re-close too, so replaying the close of a molecule's last step heals a root left open. + // + // OPT-IN, because an orchestrator may run its own molecule lifecycle. A root that its close policy refuses (a live blocker) or a close guard refuses (a template, a pinned root, a root assigned to another actor) stays open, is named in `molecule_auto_close_refusal`, and never fails this close. Any other failure of the root close, such as a storage error, fails this close with it. + AutoCloseMolecule *bool `json:"auto_close_molecule,omitempty"` + // ExpectedVersion Requires the row's revision to equal this value BEFORE the close. A miss refuses the whole request with `409 precondition_failed` and writes nothing — `UpdateIssueRequest.expected_version`'s contract, on the operation that closes one row. // // IT IS CHECKED BEFORE THE IDEMPOTENT RE-CLOSE, which is the one place this guard differs from the update's. A re-close of a row somebody else has moved since the caller read it is a `409` and not the 200-with-`already_closed` the same body earns without a guard: a replay whose premise has expired is a refusal the caller wants to see, and it is the only way `already_closed` can be trusted as "nothing has happened here since". @@ -946,9 +985,15 @@ type CloseIssueResponse struct { // AlreadyClosed True when the issue was already closed and this call changed nothing — the idempotent re-close, mirroring `ClaimResponse.already_claimed`. The response still carries the row, and `reason`/`session` were not rewritten. AlreadyClosed bool `json:"already_closed"` + // AutoClosedMolecule A tracked work item. Property semantics documented here apply to every schema that repeats them below. + AutoClosedMolecule *Issue `json:"auto_closed_molecule,omitempty"` + // Issue A tracked work item. Property semantics documented here apply to every schema that repeats them below. Issue Issue `json:"issue"` + // MoleculeAutoCloseRefusal With `issues.close.autoCloseMolecule` and `auto_close_molecule`: why the molecule root this close completed stayed open — the root's own close policy refused it (a live blocker on the root), or a close guard did (a template, a pinned root, a root assigned to another actor). Prose; absent when nothing was refused. `auto_closed_molecule`, beside it, is the root the close also closed, absent when none was. + MoleculeAutoCloseRefusal *string `json:"molecule_auto_close_refusal,omitempty"` + // OpenChildren How many open children the close observed. Reported by a FORCED close — including an idempotent re-close — because a caller that bypassed the guard is exactly the caller that wants the number. An unforced close that got this far had none, so it reports 0. OpenChildren int `json:"open_children"` @@ -972,6 +1017,9 @@ type CloseOutcome struct { // Assignee With `not_assignee`: the issue's assignee the refusing transaction observed, exactly as `Problem.assignee` carries it for the single close. Absent on a successful item and on every other refusal. Assignee *string `json:"assignee,omitempty"` + // AutoClosedMolecule A tracked work item. Property semantics documented here apply to every schema that repeats them below. + AutoClosedMolecule *Issue `json:"auto_closed_molecule,omitempty"` + // Blockers With `not_closable` on the live-blocker refusal: the blockers that refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. Blockers *[]Blocker `json:"blockers,omitempty"` @@ -989,6 +1037,9 @@ type CloseOutcome struct { // IssueId The id the caller asked for, echoed verbatim so an outcome can be read without indexing back into the request. IssueId string `json:"issue_id"` + // MoleculeAutoCloseRefusal `CloseIssueResponse.molecule_auto_close_refusal` for this item, under `issues.close.autoCloseMolecule`; `auto_closed_molecule` beside it is `CloseIssueResponse.auto_closed_molecule`. Present only on a successful item. + MoleculeAutoCloseRefusal *string `json:"molecule_auto_close_refusal,omitempty"` + // OpenChildren How many open children the transaction observed for this item. // // ITS MEANING FOLLOWS `code`, and both readings are the ones the single close already publishes. On a SUCCESSFUL item it is always present and is `CloseIssueResponse.open_children` — the number a forced close bypassed, reported even for an idempotent re-close, and `0` for an unforced close that got that far. On a REFUSED item its PRESENCE is the discriminator between the two `not_closable` refusals, exactly as it is on a problem document: present means open children, absent means a live blocker. @@ -1050,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2804,6 +2855,9 @@ type UpdateIssueJSONRequestBody = UpdateIssueRequest // AddCommentJSONRequestBody defines body for AddComment for application/json ContentType. type AddCommentJSONRequestBody = AddCommentRequest +// AdvanceMoleculeJSONRequestBody defines body for AdvanceMolecule for application/json ContentType. +type AdvanceMoleculeJSONRequestBody = AdvanceMoleculeRequest + // CompareAndSetMetadataJSONRequestBody defines body for CompareAndSetMetadata for application/json ContentType. type CompareAndSetMetadataJSONRequestBody = CompareAndSetMetadataRequest diff --git a/internal/httpapi/batch_close.go b/internal/httpapi/batch_close.go index d28bea778..abcc4aaf7 100644 --- a/internal/httpapi/batch_close.go +++ b/internal/httpapi/batch_close.go @@ -31,7 +31,7 @@ const ( // lists at each level, refused BY NAME for the reason every other body on this // surface is. var ( - batchCloseRequestMembers = []string{claimActorMember, "items", "session", "force"} + batchCloseRequestMembers = []string{claimActorMember, "items", "session", "force", autoCloseMoleculeMember} batchCloseItemMembers = []string{"id", "reason"} ) @@ -114,6 +114,8 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { if issue != nil { wire.Issue = issue } + wire.AutoClosedMolecule = outcome.AutoClosedMolecule + wire.MoleculeAutoCloseRefusal = optionalString(outcome.MoleculeAutoCloseRefusal) return wire } @@ -184,6 +186,10 @@ func (s *Server) batchCloseRequest(w http.ResponseWriter, r *http.Request) (issu if !ok { return issueops.CloseBatchRequest{}, false } + autoCloseMolecule, ok := s.booleanMember(w, r, members, autoCloseMoleculeMember) + if !ok { + return issueops.CloseBatchRequest{}, false + } items, ok := s.batchCloseItems(w, r, members) if !ok { return issueops.CloseBatchRequest{}, false @@ -192,10 +198,11 @@ func (s *Server) batchCloseRequest(w http.ResponseWriter, r *http.Request) (issu // need a second, body-shaped spelling of the ready-filter vocabulary that // listReadyWork and claimNext both express as query parameters. return issueops.CloseBatchRequest{ - Actor: actor, - Items: items, - Session: session, - Force: force, + Actor: actor, + Items: items, + Session: session, + Force: force, + AutoCloseMolecule: autoCloseMolecule, }, true } diff --git a/internal/httpapi/claim.go b/internal/httpapi/claim.go index 5b5db6326..92368aea7 100644 --- a/internal/httpapi/claim.go +++ b/internal/httpapi/claim.go @@ -340,6 +340,7 @@ type timedProvider struct { var ( _ uow.IssueReaderSource = timedProvider{} _ uow.IssueClaimerSource = timedProvider{} + _ uow.MoleculeStepperSource = timedProvider{} _ uow.BatchCloserSource = timedProvider{} _ uow.ReadyClaimerSource = timedProvider{} _ uow.ReleaserSource = timedProvider{} @@ -481,6 +482,12 @@ func (p timedProvider) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return uow.NewLeaseReclaimer(p) } +// MoleculeStepper builds the advance-a-molecule role OVER THIS WRAPPER, so +// every read and the claim it composes go through this wrapper's roles. +func (p timedProvider) MoleculeStepper() (issueops.MoleculeStepper, error) { + return uow.NewMoleculeStepper(p) +} + // IssueRelations builds the single-anchor neighbor role OVER THIS WRAPPER, for // the same reason and with the same hazard as IssueReader. It is the one // accessor here whose name is not the role's: the seam spells it IssueRelations diff --git a/internal/httpapi/close.go b/internal/httpapi/close.go index 0c1549029..bdd0ba0e8 100644 --- a/internal/httpapi/close.go +++ b/internal/httpapi/close.go @@ -17,7 +17,11 @@ import ( // schema is additionalProperties: false, so anything else is refused BY NAME — // which is why the body is decoded as raw members first, the posture claim and // batchCreate already take. -var closeRequestMembers = []string{"actor", "reason", "session", "force", expectedVersionMember} +var closeRequestMembers = []string{"actor", "reason", "session", "force", expectedVersionMember, autoCloseMoleculeMember} + +// autoCloseMoleculeMember is the request member behind +// CapIssuesCloseAutoCloseMolecule, shared by the single and the batch close. +const autoCloseMoleculeMember = "auto_close_molecule" // handleClose closes one issue: the second half of the loop this surface exists // to serve, and the half that still forked a subprocess. @@ -73,10 +77,12 @@ func (s *Server) handleClose(w http.ResponseWriter, r *http.Request) { // fill. types.Issue.RowVersion is `json:"-"`, so the Issue body cannot carry // it and this member is where it lives — updateIssue's arrangement exactly. writeJSON(w, apigen.CloseIssueResponse{ - Issue: *result.Issue, - AlreadyClosed: !result.Changed, - OpenChildren: result.OpenChildren, - Revision: types.RevisionToken(result.Issue.RowVersion), + Issue: *result.Issue, + AlreadyClosed: !result.Changed, + OpenChildren: result.OpenChildren, + Revision: types.RevisionToken(result.Issue.RowVersion), + AutoClosedMolecule: result.AutoClosedMolecule, + MoleculeAutoCloseRefusal: optionalString(result.MoleculeAutoCloseRefusal), }) } @@ -118,14 +124,19 @@ func (s *Server) closeRequest(w http.ResponseWriter, r *http.Request, id string) s.fail(w, r, *res) return issueops.CloseRequest{}, false } + autoCloseMolecule, ok := s.booleanMember(w, r, members, autoCloseMoleculeMember) + if !ok { + return issueops.CloseRequest{}, false + } return issueops.CloseRequest{ - Actor: actor, - IssueID: id, - Reason: reason, - Session: session, - Force: force, - ExpectedVersion: expectedVersion, + Actor: actor, + IssueID: id, + Reason: reason, + Session: session, + Force: force, + ExpectedVersion: expectedVersion, + AutoCloseMolecule: autoCloseMolecule, }, true } @@ -284,3 +295,11 @@ func versionPreconditionResult(expected *int64) Result { // the three operations that publish it so the member name a client reads off // `param` cannot drift from the member name it sent. const expectedVersionMember = "expected_version" + +// optionalString spells "" as an absent member. +func optionalString(value string) *string { + if value == "" { + return nil + } + return &value +} diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index fc1a38b4a..fb71323f7 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -443,6 +443,9 @@ const ( // the reason and session under first-close-wins, the done-status // normalization, and the close policy vocabulary. OpCloseIssue = "closeIssue" + // OpAdvanceMolecule moves a molecule on after one of its steps closed: the + // operation behind `bd close --continue` (issueops.MoleculeStepper). + OpAdvanceMolecule = "advanceMolecule" // OpReopenIssue is the close's mirror, and it completes the lifecycle pair // so a recovery flow works end to end over this surface. It is the one write // here with no POLICY conflict code: reopen takes an issue OUT of the done @@ -899,6 +902,13 @@ var operationCodes = map[string][]Code{ CodeTemplateReadOnly, CodeIssuePinned, CodeNotAssignee, CodeBusy, CodeDBUnavailable, CodeInternal, }, + // No conflict code: a ready step another actor holds, or one moved out of + // a claimable status, is skipped for the next one rather than refused — + // the role's own rule — so the only 404 is the path id's. + OpAdvanceMolecule: { + CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, + CodeBusy, CodeDBUnavailable, CodeInternal, + }, // ONE 409, and it is a PRECONDITION rather than a policy. Close has a policy // guard — open children, a live blocker — and reopen is the direction that // takes an issue OUT of the done category rather than putting it in, so diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 7060cdd51..e4f6519cc 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -204,6 +204,16 @@ const CapExternalDependencies = "policy.external_dependencies" // lists them; advertisedCapabilities adds each one whose condition holds. var conditionalCapabilities = []string{CapExternalDependencies} +// CapIssuesCloseAutoCloseMolecule is the behavior capability that advertises +// CloseIssueRequest.auto_close_molecule and BatchCloseRequest.auto_close_molecule +// with their paired response members (auto_closed_molecule, +// molecule_auto_close_refusal): close a completed molecule's root in the same +// transaction as the step close (issueops.CloseRequest.AutoCloseMolecule). An +// older server answers the request member with `400 +// invalid_argument`/`unknown_parameter`; the client checks this token before +// sending it. +const CapIssuesCloseAutoCloseMolecule = "issues.close.autoCloseMolecule" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -759,6 +769,16 @@ var routeTable = []route{ implemented: true, handler: (*Server).handleReopen, }, + { + op: OpAdvanceMolecule, + method: http.MethodPost, + pattern: customMethodPattern, + specPath: "/v0/beads/issues/{id}:advanceMolecule", + customMethod: ":advanceMolecule", + capability: "issues.advanceMolecule", + implemented: true, + handler: (*Server).handleAdvanceMolecule, + }, { op: OpBatchCloseIssues, method: http.MethodPost, @@ -966,6 +986,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, + CapIssuesCloseAutoCloseMolecule, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 4a1791b45..c4e72600a 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -2408,3 +2408,23 @@ func logValue(v any) string { func millis(d time.Duration) string { return strconv.FormatFloat(float64(d)/float64(time.Millisecond), 'f', 3, 64) } + +// moleculeStepper returns the advance-a-molecule surface for one request. +// +// On the store-shaped source there is no configured field for it: the role is +// issueops.NewMoleculeStepper composed over roles the source already carries +// (batch getter, relations, edge reader, claimer, lifecycle), so building it +// here from those is the same composition a store's own accessor makes. On the +// provider it is built over the timed wrapper, the same two-step every role +// above performs. +func (s *Server) moleculeStepper(r *http.Request) (issueops.MoleculeStepper, error) { + if s.provider == nil { + return issueops.NewMoleculeStepper(issueops.MoleculeStepperRoles{ + Reader: issueops.MoleculeReader{BatchGetter: s.issueBatchGetter, Relations: s.issueRelations, EdgeReader: s.issueEdges}, + Claimer: checkedClaimer{inner: s.issueClaimer}, + Lifecycle: checkedLifecycle{inner: s.issueLifecycle}, + }), nil + } + var src uow.MoleculeStepperSource = timedProvider{inner: s.provider, rec: requestInfo(r.Context())} + return src.MoleculeStepper() +} diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 0170ebb61..7210f3d98 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -637,7 +637,8 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", - "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority") + "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", + "issues.close.autoCloseMolecule") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index d95723cb0..8dc0099d6 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -3340,6 +3340,64 @@ paths: '503': $ref: '#/components/responses/Unavailable' + /v0/beads/issues/{id}:advanceMolecule: + post: + operationId: advanceMolecule + summary: Advance a molecule to its next ready step + description: >- + The operation behind `bd close --continue`: after the step this path + names closed, find where its molecule stands and, with `auto_claim`, + claim the next ready step for `actor`. + + + A step is READY when it is open and no open step of the same molecule + blocks it (blocks, conditional-blocks or waits-for). The claim is a + real claim — the step's assignee becomes `actor` — and a step another + actor holds is skipped, never taken: a ready step lost that way, or one + moved out of a claimable status since it was read, falls through to + the next ready step. When every ready step is lost nothing is claimed + and `next_step` names the first. + + + A step in no molecule answers 200 with no `molecule_id`. A step id that + names nothing is `404`. + parameters: + - $ref: '#/components/parameters/IssueID' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AdvanceMoleculeRequest' + security: + - bearerToken: [] + responses: + '200': + description: Where the molecule stands, and what was claimed. + content: + application/json: + schema: + $ref: '#/components/schemas/AdvanceMoleculeResponse' + '400': + description: >- + Invalid request: an unknown query parameter, an unparseable or + oversized body, an unknown body member, or an `actor` that is empty + after trimming, longer than 256 bytes, or carrying control + characters. + x-bd-codes: [invalid_argument] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '401': + $ref: '#/components/responses/Unauthenticated' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + /v0/beads/issues/{id}:reopen: post: operationId: reopenIssue @@ -8104,7 +8162,8 @@ components: `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, - `issues.casMetadata`; the behavior tokens are `project.enforce`, + `issues.casMetadata`, `issues.advanceMolecule`; the behavior + tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that @@ -8148,7 +8207,13 @@ components: `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores - the default P2 rather than P0. One behavior token is + the default P2 rather than P0, and + `issues.close.autoCloseMolecule`, which announces that + `POST /v0/beads/issues/{id}:close` and + `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` + and answer `auto_closed_molecule` and + `molecule_auto_close_refusal` rather than a `400` naming the + member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's @@ -11155,6 +11220,21 @@ components: and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. + auto_close_molecule: + type: boolean + default: false + description: >- + Behind the `issues.close.autoCloseMolecule` behavior token: an + older server answers `unknown_parameter` for this name. + + + Close each item's molecule root when the batch completed it: + `CloseIssueRequest.auto_close_molecule` for every item that closed + or was already closed, run after all the closes inside the same + transaction. A batch that closes several steps of one molecule + decides its root once, and the root it closed, or its refusal, is + reported on the outcome of the first successful item under that + root. BatchCloseItem: type: object @@ -11276,6 +11356,15 @@ components: Prose for a refusal, never load-bearing and present only with `code`. It reflects the request and this server's own words rather than the role's message, for the reason `Problem.detail` gives. + auto_closed_molecule: + $ref: '#/components/schemas/Issue' + molecule_auto_close_refusal: + type: string + description: >- + `CloseIssueResponse.molecule_auto_close_refusal` for this item, + under `issues.close.autoCloseMolecule`; `auto_closed_molecule` + beside it is `CloseIssueResponse.auto_closed_molecule`. Present + only on a successful item. Blocker: type: object @@ -11302,6 +11391,47 @@ components: `conditional-blocks`). ABSENT when the refusal did not report it, which is the case for an `external:` blocker. + AdvanceMoleculeRequest: + type: object + additionalProperties: false + required: [actor] + properties: + actor: + type: string + minLength: 1 + maxLength: 256 + pattern: '^[^\u0000-\u001F\u007F-\u009F\u2028\u2029]+$' + description: >- + Who advances, and with `auto_claim` the claimed step's assignee. + `ClaimRequest.actor`'s rules exactly. + auto_claim: + type: boolean + default: false + description: >- + Claim the next ready step for `actor`. Without it the operation + writes nothing and only names the step. + AdvanceMoleculeResponse: + type: object + required: [closed_step, complete, claimed] + properties: + closed_step: + $ref: '#/components/schemas/Issue' + molecule_id: + type: string + description: >- + The step's molecule root. ABSENT when the step belongs to no + molecule, and then nothing else below is set. + complete: + type: boolean + description: >- + Every step of the molecule is closed; `next_step` is then absent. + next_step: + $ref: '#/components/schemas/Issue' + claimed: + type: boolean + description: >- + `next_step` was claimed for `actor`, and is the post-claim row. + CloseIssueRequest: type: object additionalProperties: false @@ -11387,6 +11517,32 @@ components: member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. + auto_close_molecule: + type: boolean + default: false + description: >- + Behind the `issues.close.autoCloseMolecule` behavior token: an + older server answers `unknown_parameter` for this name, and a + client checks the token before sending it. + + + Close the issue's molecule root IN THE SAME TRANSACTION when this + close completed it: the root is an auto-closing molecule root (a + `molecule`-typed or ephemeral root, or a template-labeled epic), + it is open, and every step under it is now closed. The root close + is unforced, guarded on the root revision read inside that + transaction, and records `all steps complete` and this request's + `session`. It runs for an idempotent re-close too, so replaying + the close of a molecule's last step heals a root left open. + + + OPT-IN, because an orchestrator may run its own molecule + lifecycle. A root that its close policy refuses (a live blocker) + or a close guard refuses (a template, a pinned root, a root + assigned to another actor) stays open, is named in + `molecule_auto_close_refusal`, and never fails this close. Any + other failure of the root close, such as a storage error, fails + this close with it. CloseIssueResponse: type: object required: [issue, already_closed, open_children, revision] @@ -11431,6 +11587,18 @@ components: `float` — rounds anything past 2^53 to a value NEAR the token that is not it, so a guard composed from it is refused against a row nothing else touched. A string round-trips exactly in every consumer. + auto_closed_molecule: + $ref: '#/components/schemas/Issue' + molecule_auto_close_refusal: + type: string + description: >- + With `issues.close.autoCloseMolecule` and `auto_close_molecule`: + why the molecule root this close completed stayed open — the + root's own close policy refused it (a live blocker on the root), + or a close guard did (a template, a pinned root, a root assigned + to another actor). Prose; absent when nothing was refused. `auto_closed_molecule`, + beside it, is the root the close also closed, absent when none + was. ReopenIssueRequest: type: object additionalProperties: false diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 02a66f58c..6604685ac 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -33,6 +33,48 @@ "required": true, "item_type": "object" }, + { + "schema": "AdvanceMoleculeRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "AdvanceMoleculeRequest", + "member": "auto_claim", + "type": "boolean", + "required": false + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "claimed", + "type": "boolean", + "required": true + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "closed_step", + "type": "object", + "required": true + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "complete", + "type": "boolean", + "required": true + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "molecule_id", + "type": "string", + "required": false + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "next_step", + "type": "object", + "required": false + }, { "schema": "AnchorEdgeCount", "member": "count", @@ -595,6 +637,12 @@ "type": "string", "required": true }, + { + "schema": "BatchCloseRequest", + "member": "auto_close_molecule", + "type": "boolean", + "required": false + }, { "schema": "BatchCloseRequest", "member": "force", @@ -1166,6 +1214,12 @@ "type": "string", "required": true }, + { + "schema": "CloseIssueRequest", + "member": "auto_close_molecule", + "type": "boolean", + "required": false + }, { "schema": "CloseIssueRequest", "member": "expected_version", @@ -1196,12 +1250,24 @@ "type": "boolean", "required": true }, + { + "schema": "CloseIssueResponse", + "member": "auto_closed_molecule", + "type": "object", + "required": false + }, { "schema": "CloseIssueResponse", "member": "issue", "type": "object", "required": true }, + { + "schema": "CloseIssueResponse", + "member": "molecule_auto_close_refusal", + "type": "string", + "required": false + }, { "schema": "CloseIssueResponse", "member": "open_children", @@ -1226,6 +1292,12 @@ "type": "string", "required": false }, + { + "schema": "CloseOutcome", + "member": "auto_closed_molecule", + "type": "object", + "required": false + }, { "schema": "CloseOutcome", "member": "blockers", @@ -1257,6 +1329,12 @@ "type": "string", "required": true }, + { + "schema": "CloseOutcome", + "member": "molecule_auto_close_refusal", + "type": "string", + "required": false + }, { "schema": "CloseOutcome", "member": "open_children", @@ -4866,6 +4944,13 @@ "required": true, "style": "simple" }, + { + "schema": "param:advanceMolecule", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, { "schema": "param:claimIssue", "member": "path:id", @@ -6106,6 +6191,8 @@ "AddCommentRequest": "request", "AddDependenciesRequest": "request", "AddDependenciesResponse": "response", + "AdvanceMoleculeRequest": "request", + "AdvanceMoleculeResponse": "response", "AnchorEdgeCount": "response", "ApplyBatchRequest": "request", "ApplyBatchResponse": "response", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index ea7f30ec2..265a370e8 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -32,6 +32,7 @@ go_library( "memories.go", "metadata.go", "metadatacas.go", + "molecule.go", "offrole.go", "open_with.go", "order.go", @@ -117,6 +118,7 @@ go_test( "lifecycle_update_test.go", "list_sort_test.go", "metadatacas_test.go", + "molecule_test.go", "open_with_test.go", "race_detector_off_test.go", "race_detector_on_test.go", @@ -149,6 +151,7 @@ go_test( "served_list_test.go", "served_memories_deps_test.go", "served_metadata_cas_test.go", + "served_molecule_test.go", "served_project_swap_test.go", "served_querier_test.go", "served_reader_test.go", diff --git a/internal/httpclient/batchcloser.go b/internal/httpclient/batchcloser.go index d7257186a..17a296eb4 100644 --- a/internal/httpclient/batchcloser.go +++ b/internal/httpclient/batchcloser.go @@ -123,6 +123,11 @@ func (b *httpBatchCloser) CloseBatch(ctx context.Context, req issueops.CloseBatc if err != nil { return issueops.CloseBatchResult{}, err } + // AutoCloseMolecule rides the member only where the server advertises it; + // elsewhere the batch is refused before it is dialed (molecule.go). + if err := b.store.requireAutoCloseMolecule(ctx, "BatchCloser.CloseBatch", req.AutoCloseMolecule); err != nil { + return issueops.CloseBatchResult{}, err + } if served { return b.serveBatch(ctx, req) } @@ -175,7 +180,12 @@ func (b *httpBatchCloser) serveComposedSingle(ctx context.Context, req issueops. return issueops.CloseBatchResult{}, err } item := req.Items[0] - res, err := b.wire.CloseIssue(ctx, item.IssueID, closeBody(req.Actor, item.Reason, req.Session, req.Force)) + body := closeBody(req.Actor, item.Reason, req.Session, req.Force) + if req.AutoCloseMolecule { + autoClose := true + body.AutoCloseMolecule = &autoClose + } + res, err := b.wire.CloseIssue(ctx, item.IssueID, body) if err != nil { if outcome, ok := perItemCloseRefusal(item.IssueID, err); ok { // A per-item refusal is a RESULT and never the method's error. The @@ -188,10 +198,12 @@ func (b *httpBatchCloser) serveComposedSingle(ctx context.Context, req issueops. issue := res.Issue return issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{{ - IssueID: item.IssueID, - Issue: &issue, - Changed: !res.AlreadyClosed, - OpenChildren: res.OpenChildren, + IssueID: item.IssueID, + Issue: &issue, + Changed: !res.AlreadyClosed, + OpenChildren: res.OpenChildren, + AutoClosedMolecule: res.AutoClosedMolecule, + MoleculeAutoCloseRefusal: derefStringPtr(res.MoleculeAutoCloseRefusal), }}}, nil } @@ -218,6 +230,10 @@ func batchCloseBody(req issueops.CloseBatchRequest) apigen.BatchCloseRequest { force := true body.Force = &force } + if req.AutoCloseMolecule { + autoClose := true + body.AutoCloseMolecule = &autoClose + } return body } @@ -265,6 +281,10 @@ func decodeBatchCloseOutcome(actor string, item apigen.CloseOutcome) issueops.Cl out.OpenChildren = *item.OpenChildren } out.Changed = item.AlreadyClosed == nil || !*item.AlreadyClosed + out.AutoClosedMolecule = item.AutoClosedMolecule + if item.MoleculeAutoCloseRefusal != nil { + out.MoleculeAutoCloseRefusal = *item.MoleculeAutoCloseRefusal + } return out } diff --git a/internal/httpclient/batchgetter.go b/internal/httpclient/batchgetter.go index 3f48ef403..c1f6eccd3 100644 --- a/internal/httpclient/batchgetter.go +++ b/internal/httpclient/batchgetter.go @@ -4,6 +4,7 @@ package httpclient import ( "context" + "errors" "fmt" "net/http" @@ -66,6 +67,15 @@ func (g *httpBatchGetter) GetMany(ctx context.Context, req issueops.GetManyReque return issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}}, nil } + token, _ := wire.CapabilityFor(wire.OpBatchGetIssues) + served, err := g.store.servesCapability(ctx, token) + if err != nil { + return issueops.GetManyResult{}, err + } + if !served { + return g.getOneAtATime(ctx, req.IDs) + } + var body apigen.BatchGetIssuesResult if err := g.store.dispatch(ctx, wire.Request{ Op: wire.OpBatchGetIssues, @@ -128,3 +138,39 @@ func checkGetManyIDs(ids []string) error { } return nil } + +// getOneAtATime answers a GetMany against a server that does not advertise +// issues.batchGet with one getIssue per distinct id, in first-mention order — +// the same rows and the same Missing a batch would answer, without its single +// snapshot. It is a transport fallback for a capability, kept here so no +// caller of the role has to know which servers publish the batch. +func (g *httpBatchGetter) getOneAtATime(ctx context.Context, ids []string) (issueops.GetManyResult, error) { + reader, err := g.store.IssueReader() + if err != nil { + return issueops.GetManyResult{}, err + } + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + seen := make(map[string]bool, len(ids)) + for _, id := range ids { + if seen[id] { + continue + } + seen[id] = true + details, err := reader.Get(ctx, issueops.GetRequest{ID: id}) + if errors.Is(err, issueops.ErrNotFound) { + out.Missing = append(out.Missing, id) + continue + } + if err != nil { + return issueops.GetManyResult{}, err + } + issue := details.Issue + if len(issue.Labels) == 0 && len(details.Labels) > 0 { + issue.Labels = append([]string(nil), details.Labels...) + } + // Hydration is labels only, as the batch's is. + issue.Dependencies, issue.Comments = nil, nil + out.Issues = append(out.Issues, &issue) + } + return out, nil +} diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 0caefe063..ea0df8554 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -464,6 +464,15 @@ func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (i body := closeBody(req.Actor, req.Reason, req.Session, req.Force) body.ExpectedVersion = revisionGuard(req.ExpectedVersion) + // AutoCloseMolecule rides the member only where the server advertises it; + // elsewhere the close is refused before it is dialed (molecule.go). + if err := l.store.requireAutoCloseMolecule(ctx, "IssueLifecycle.Close", req.AutoCloseMolecule); err != nil { + return issueops.CloseResult{}, err + } + if req.AutoCloseMolecule { + autoClose := true + body.AutoCloseMolecule = &autoClose + } res, err := l.wire.CloseIssue(ctx, req.IssueID, body) if err != nil { return issueops.CloseResult{}, err @@ -474,11 +483,16 @@ func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (i if issue.RowVersion, err = parseRevision("closeIssue", res.Revision); err != nil { return issueops.CloseResult{}, err } - return issueops.CloseResult{ - Issue: &issue, - Changed: !res.AlreadyClosed, - OpenChildren: res.OpenChildren, - }, nil + result := issueops.CloseResult{ + Issue: &issue, + Changed: !res.AlreadyClosed, + OpenChildren: res.OpenChildren, + AutoClosedMolecule: res.AutoClosedMolecule, + } + if res.MoleculeAutoCloseRefusal != nil { + result.MoleculeAutoCloseRefusal = *res.MoleculeAutoCloseRefusal + } + return result, nil } // Reopen dials POST issues/{id}:reopen. diff --git a/internal/httpclient/molecule.go b/internal/httpclient/molecule.go new file mode 100644 index 000000000..a9571345b --- /dev/null +++ b/internal/httpclient/molecule.go @@ -0,0 +1,125 @@ +package httpclient + +import ( + "context" + "net/http" + "slices" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// servesCapability reports whether the server advertises token. It forces the +// one lazy handshake the store owns and returns that handshake's error rather +// than reading a failure as "not served": every caller below is a write whose +// two legs both dial, so a handshake that cannot be had means the write cannot +// be attempted at all (servesBatchClose's reasoning). +func (s *Store) servesCapability(ctx context.Context, token string) (bool, error) { + snap, err := s.snapshot(ctx) + if err != nil { + return false, err + } + return snap != nil && slices.Contains(snap.Capabilities, token), nil +} + +// ServesMoleculeAutoClose implements storage.MoleculeAutoCloseProber from the +// handshake: whether this server runs CloseRequest.AutoCloseMolecule. +func (s *Store) ServesMoleculeAutoClose(ctx context.Context) (bool, error) { + return s.servesCapability(ctx, wire.CapCloseAutoCloseMolecule) +} + +var _ storage.MoleculeAutoCloseProber = (*Store)(nil) + +// ServesMoleculeAdvance implements storage.MoleculeAdvanceProber from the +// handshake: whether this server serves issues.advanceMolecule. +func (s *Store) ServesMoleculeAdvance(ctx context.Context) (bool, error) { + token, _ := wire.CapabilityFor(wire.OpAdvanceMolecule) + return s.servesCapability(ctx, token) +} + +var _ storage.MoleculeAdvanceProber = (*Store)(nil) + +// requireAutoCloseMolecule refuses, before anything is dialed, a close that +// asks for AutoCloseMolecule against a server that does not advertise +// wire.CapCloseAutoCloseMolecule. The rule runs in ONE place — the server's +// library, inside the step close's transaction — so the client never runs a +// second, client-side copy of it, and it never drops the member and closes the +// step without the molecule either. The refusal is the typed capability error +// (Unsup.Capability names the token). +func (s *Store) requireAutoCloseMolecule(ctx context.Context, op string, requested bool) error { + if !requested { + return nil + } + served, err := s.servesCapability(ctx, wire.CapCloseAutoCloseMolecule) + if err != nil { + return err + } + if !served { + return s.unsupportedCapability(op, wire.CapCloseAutoCloseMolecule) + } + return nil +} + +// MoleculeStepper serves the advance-a-molecule role on +// POST /v0/beads/issues/{id}:advanceMolecule. +func (s *Store) MoleculeStepper() (issueops.MoleculeStepper, error) { + if _, err := s.roleWire("MoleculeStepper"); err != nil { + return nil, err + } + return &httpMoleculeStepper{store: s}, nil +} + +// httpMoleculeStepper dials advanceMolecule, where the server's library runs +// the one stepper. Against a server that does not advertise +// issues.advanceMolecule it refuses before dialing with the typed capability +// error; it never composes a client-side copy of the advance. +type httpMoleculeStepper struct{ store *Store } + +var _ issueops.MoleculeStepper = (*httpMoleculeStepper)(nil) + +func (m *httpMoleculeStepper) Advance(ctx context.Context, req issueops.AdvanceRequest) (issueops.AdvanceResult, error) { + if err := issueops.ValidateAdvanceRequest(req); err != nil { + return issueops.AdvanceResult{}, err + } + token, _ := wire.CapabilityFor(wire.OpAdvanceMolecule) + served, err := m.store.servesCapability(ctx, token) + if err != nil { + return issueops.AdvanceResult{}, err + } + if !served { + return issueops.AdvanceResult{}, m.store.unsupportedCapability("MoleculeStepper.Advance", token) + } + + path, err := wire.IssueMethodPath(req.ClosedStepID, wire.MethodAdvanceMolecule) + if err != nil { + return issueops.AdvanceResult{}, err + } + body := apigen.AdvanceMoleculeRequest{Actor: req.Actor} + if req.AutoClaim { + autoClaim := true + body.AutoClaim = &autoClaim + } + var res apigen.AdvanceMoleculeResponse + if err := m.store.dispatch(ctx, wire.Request{ + Op: wire.OpAdvanceMolecule, + Method: http.MethodPost, + Path: path, + Body: body, + IssueID: req.ClosedStepID, + }, &res); err != nil { + return issueops.AdvanceResult{}, err + } + closed := res.ClosedStep + out := issueops.AdvanceResult{ + ClosedStep: &closed, + Complete: res.Complete, + NextStep: res.NextStep, + Claimed: res.Claimed, + } + if res.MoleculeId != nil { + out.MoleculeID = *res.MoleculeId + } + return out, nil +} diff --git a/internal/httpclient/molecule_test.go b/internal/httpclient/molecule_test.go new file mode 100644 index 000000000..eb21a9ebb --- /dev/null +++ b/internal/httpclient/molecule_test.go @@ -0,0 +1,180 @@ +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The molecule rules run in ONE place, the server's library. Against a server +// that does not advertise them the client refuses BEFORE dialing with the +// typed capability error: it never drops auto_close_molecule and closes the +// step alone, and never runs a client-side copy of the auto-close or advance. + +func requireCapabilityRefusal(t *testing.T, err error, token string, w *stubWire) { + t.Helper() + if err == nil { + t.Fatal("no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != token { + t.Errorf("Capability = %q, want %q", unsup.Capability, token) + } + if len(w.calls) != 0 { + t.Errorf("dialed %v, want none: a pre-dial refusal must never reach the wire", w.calls) + } +} + +func TestAutoCloseMoleculeRefusesBeforeDialingWhenUnadvertised(t *testing.T) { + close := issueops.CloseRequest{Actor: "alice", IssueID: "bd-1", AutoCloseMolecule: true} + + t.Run("Close", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + _, err = lifecycle.Close(t.Context(), close) + requireCapabilityRefusal(t, err, wire.CapCloseAutoCloseMolecule, w) + }) + + for name, caps := range map[string][]string{ + "CloseBatch served": {"issues.close", "issues.batchClose"}, + "CloseBatch composed": {"issues.close"}, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + closer, err := New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: caps}).BatchCloser() + if err != nil { + t.Fatal(err) + } + _, err = closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "alice", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}, AutoCloseMolecule: true, + }) + requireCapabilityRefusal(t, err, wire.CapCloseAutoCloseMolecule, w) + }) + } + + t.Run("without the flag a close still dials", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + plain := close + plain.AutoCloseMolecule = false + if _, err := lifecycle.Close(t.Context(), plain); err != nil { + t.Fatalf("Close: %v", err) + } + if w.lastClose.AutoCloseMolecule != nil { + t.Errorf("auto_close_molecule = %v, want absent when not asked for", *w.lastClose.AutoCloseMolecule) + } + }) +} + +func TestAutoCloseMoleculeRidesTheMemberWhenAdvertised(t *testing.T) { + snap := &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{ + "issues.close", "issues.batchClose", wire.CapCloseAutoCloseMolecule, + }} + root := types.Issue{ID: "bd-root", Status: types.StatusClosed} + + t.Run("Close", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0", AutoClosedMolecule: &root}} + lifecycle, err := New(testTarget(t), w, snap).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + res, err := lifecycle.Close(t.Context(), issueops.CloseRequest{Actor: "alice", IssueID: "bd-1", AutoCloseMolecule: true}) + if err != nil { + t.Fatalf("Close: %v", err) + } + if w.lastClose.AutoCloseMolecule == nil || !*w.lastClose.AutoCloseMolecule { + t.Errorf("auto_close_molecule = %v, want true", w.lastClose.AutoCloseMolecule) + } + if res.AutoClosedMolecule == nil || res.AutoClosedMolecule.ID != "bd-root" { + t.Errorf("AutoClosedMolecule = %+v, want bd-root", res.AutoClosedMolecule) + } + }) + + t.Run("CloseBatch", func(t *testing.T) { + w := &stubWire{batchClose: &apigen.BatchCloseResponse{Outcomes: []apigen.CloseOutcome{{ + IssueId: "bd-1", Issue: &types.Issue{ID: "bd-1", Status: types.StatusClosed}, AutoClosedMolecule: &root, + }}}} + closer, err := New(testTarget(t), w, snap).BatchCloser() + if err != nil { + t.Fatal(err) + } + res, err := closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "alice", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}, AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + if w.lastBatchClose.AutoCloseMolecule == nil || !*w.lastBatchClose.AutoCloseMolecule { + t.Errorf("auto_close_molecule = %v, want true", w.lastBatchClose.AutoCloseMolecule) + } + if len(res.Outcomes) != 1 || res.Outcomes[0].AutoClosedMolecule == nil || res.Outcomes[0].AutoClosedMolecule.ID != "bd-root" { + t.Errorf("Outcomes = %+v, want bd-1 reporting bd-root auto-closed", res.Outcomes) + } + }) +} + +func TestAdvanceMoleculeRefusesBeforeDialingWhenUnadvertised(t *testing.T) { + token, ok := wire.CapabilityFor(wire.OpAdvanceMolecule) + if !ok || token == "" { + t.Fatal("advanceMolecule has no capability token") + } + w := &stubWire{} + stepper, err := stubStore(t, w).MoleculeStepper() + if err != nil { + t.Fatal(err) + } + _, err = stepper.Advance(t.Context(), issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "bd-1", AutoClaim: true}) + requireCapabilityRefusal(t, err, token, w) +} + +// The CLI asks for the auto-close only where the route serves it +// (storage.ServesMoleculeAutoClose): a server silent on the token owns the +// molecule's auto-close, and a close there goes out without the member. +func TestServesMoleculeAutoCloseFollowsTheHandshake(t *testing.T) { + ctx := t.Context() + silent := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close"}}) + if storage.ServesMoleculeAutoClose(ctx, silent) { + t.Error("ServesMoleculeAutoClose = true against a server silent on the token") + } + served := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close", wire.CapCloseAutoCloseMolecule}}) + if !storage.ServesMoleculeAutoClose(ctx, served) { + t.Error("ServesMoleculeAutoClose = false against a server advertising the token") + } +} + +// `bd close --continue` asks storage.ServesMoleculeAdvance before it closes, +// so a server silent on issues.advanceMolecule is refused up front instead of +// after the step's close has committed. A handshake that fails is that error, +// not a yes: answering true would close the step and then fail the advance, +// the half-done state the probe exists to prevent. +func TestServesMoleculeAdvanceFollowsTheHandshake(t *testing.T) { + ctx := t.Context() + token, _ := wire.CapabilityFor(wire.OpAdvanceMolecule) + silent := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close"}}) + if served, err := storage.ServesMoleculeAdvance(ctx, silent); served || err != nil { + t.Errorf("ServesMoleculeAdvance = (%v, %v) against a server silent on the token, want (false, nil)", served, err) + } + advertised := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close", token}}) + if served, err := storage.ServesMoleculeAdvance(ctx, advertised); !served || err != nil { + t.Errorf("ServesMoleculeAdvance = (%v, %v) against a server advertising the token, want (true, nil)", served, err) + } + boom := errors.New("handshake failed") + failed := New(testTarget(t), handshakeWire{stubWire: &stubWire{}, err: boom}, nil) + if served, err := storage.ServesMoleculeAdvance(ctx, failed); served || !errors.Is(err, boom) { + t.Errorf("ServesMoleculeAdvance = (%v, %v) when the handshake fails, want (false, the handshake's error)", served, err) + } +} diff --git a/internal/httpclient/roles_graph.go b/internal/httpclient/roles_graph.go index 41f831959..3a0dda6a5 100644 --- a/internal/httpclient/roles_graph.go +++ b/internal/httpclient/roles_graph.go @@ -168,7 +168,35 @@ func (r httpEdgeReader) ReadEdges(ctx context.Context, req issueops.EdgeReadRequ if len(ids) == 0 { return issueops.EdgeReadResult{Anchors: []issueops.AnchorEdges{}}, nil } + // The ROLE has no anchor cap; the OPERATION refuses more than + // maxEdgeReadAnchors per request. Splitting is therefore this transport's + // job, never the caller's: each chunk is one round trip and the answers + // concatenate in request order. A read loses nothing by being split — the + // one property it gives up is the single snapshot across chunks, which + // per-anchor answers do not rely on. + if len(ids) > maxEdgeReadAnchors { + anchors := make([]issueops.AnchorEdges, 0, len(ids)) + for start := 0; start < len(ids); start += maxEdgeReadAnchors { + end := min(start+maxEdgeReadAnchors, len(ids)) + part, err := r.readEdgeChunk(ctx, ids[start:end], req.Types) + if err != nil { + return issueops.EdgeReadResult{}, err + } + anchors = append(anchors, part.Anchors...) + } + return issueops.EdgeReadResult{Anchors: anchors}, nil + } + return r.readEdgeChunk(ctx, ids, req.Types) +} +// maxEdgeReadAnchors is the listDependencies operation's cap on `issue_id` +// values (maxDependencyAnchors in internal/httpapi), redeclared because a +// client does not import the server. +const maxEdgeReadAnchors = 100 + +// readEdgeChunk answers at most maxEdgeReadAnchors distinct ids in one call. +func (r httpEdgeReader) readEdgeChunk(ctx context.Context, ids []string, edgeTypes []types.DependencyType) (issueops.EdgeReadResult, error) { + req := issueops.EdgeReadRequest{IDs: ids, Types: edgeTypes} q := url.Values{} for _, id := range ids { q.Add("issue_id", id) diff --git a/internal/httpclient/served_molecule_test.go b/internal/httpclient/served_molecule_test.go new file mode 100644 index 000000000..54b67d876 --- /dev/null +++ b/internal/httpclient/served_molecule_test.go @@ -0,0 +1,235 @@ +//go:build cgo + +package httpclient + +import ( + "context" + "errors" + "fmt" + "slices" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The molecule contract against the served surface: the client sends +// auto_close_molecule and dials advanceMolecule, so the server's library runs +// the auto-close in the step close's transaction and the advance server-side. +// +// Against a server whose handshake lacks the two molecule tokens (an older +// server) the client refuses before dialing; the served refusal test below +// pins that the step it was asked to close stays open and unclaimed. + +func moleculeContractCases() []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) +} { + return []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } +} + +func newServedMoleculeFixture(t *testing.T, env *servedEnv, subject *Store) conformance.MoleculeFixture { + t.Helper() + lifecycle, err := subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + closer, err := subject.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + stepper, err := subject.MoleculeStepper() + if err != nil { + t.Fatalf("MoleculeStepper(): %v", err) + } + return conformance.MoleculeFixture{ + IssuePrefix: env.prefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + AddDependency: env.addDependency, + QueryScalar: env.queryScalar, + } +} + +// degradedSubject is a client of env's server whose handshake lacks tokens, as +// an older server's would. +func degradedSubject(t *testing.T, env *servedEnv, strike ...string) *Store { + t.Helper() + snapshot, err := env.subject.snapshot(t.Context()) + if err != nil { + t.Fatalf("handshake: %v", err) + } + for _, token := range strike { + if !slices.Contains(snapshot.Capabilities, token) { + t.Fatalf("the served build does not advertise %q; the degraded arm would prove nothing", token) + } + } + masked := *snapshot + masked.Capabilities = slices.DeleteFunc(slices.Clone(snapshot.Capabilities), func(token string) bool { + return slices.Contains(strike, token) + }) + return New(env.subject.target, env.subject.wire, &masked) +} + +func advanceToken(t *testing.T) string { + t.Helper() + token, ok := wire.CapabilityFor(wire.OpAdvanceMolecule) + if !ok || token == "" { + t.Fatal("advanceMolecule has no capability token") + } + return token +} + +func TestServedMoleculeContract(t *testing.T) { + for _, tc := range moleculeContractCases() { + t.Run(tc.name, func(t *testing.T) { + env := newServedEnv(t, "hmol") + tc.run(t, t.Context(), newServedMoleculeFixture(t, env, env.subject)) + }) + } +} + +// Against an older server (the molecule tokens struck from the handshake) a +// close that asks for the auto-close and an advance are refused before they +// are dialed: the step stays open, nothing is claimed, and the refusal names +// the missing token. +func TestServedMoleculeRefusesAgainstAnOlderServer(t *testing.T) { + env := newServedEnv(t, "hmold") + ctx := t.Context() + subject := degradedSubject(t, env, wire.CapCloseAutoCloseMolecule, advanceToken(t)) + fixture := newServedMoleculeFixture(t, env, subject) + for _, issue := range []*types.Issue{ + {ID: "hmold-r", Title: "root", IssueType: types.TypeMolecule, Status: types.StatusOpen}, + {ID: "hmold-r.1", Title: "step", IssueType: types.TypeTask, Status: types.StatusOpen}, + {ID: "hmold-r.2", Title: "next", IssueType: types.TypeTask, Status: types.StatusOpen}, + } { + if err := env.createIssue(ctx, issue, "t"); err != nil { + t.Fatal(err) + } + } + for _, child := range []string{"hmold-r.1", "hmold-r.2"} { + if err := env.addDependency(ctx, &types.Dependency{IssueID: child, DependsOnID: "hmold-r", Type: types.DepParentChild}, "t"); err != nil { + t.Fatal(err) + } + } + status := func(id string) string { + t.Helper() + got, err := env.subject.GetIssue(ctx, id) + if err != nil { + t.Fatal(err) + } + return string(got.Status) + "/" + got.Assignee + } + requireRefusal := func(what string, err error, token string) { + t.Helper() + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Capability != token { + t.Fatalf("%s = %v, want the pre-dial refusal naming %q", what, err, token) + } + } + + _, err := fixture.Lifecycle.Close(ctx, issueops.CloseRequest{Actor: "alice", IssueID: "hmold-r.1", AutoCloseMolecule: true}) + requireRefusal("Close", err, wire.CapCloseAutoCloseMolecule) + _, err = fixture.BatchCloser.CloseBatch(ctx, issueops.CloseBatchRequest{ + Actor: "alice", Items: []issueops.BatchCloseItem{{IssueID: "hmold-r.1"}}, AutoCloseMolecule: true, + }) + requireRefusal("CloseBatch", err, wire.CapCloseAutoCloseMolecule) + if got := status("hmold-r.1"); got != "open/" { + t.Errorf("the refused close's step = %s, want open/ (nothing dialed)", got) + } + + _, err = fixture.Stepper.Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "hmold-r.1", AutoClaim: true}) + requireRefusal("Advance", err, advanceToken(t)) + if got := status("hmold-r.2"); got != "open/" { + t.Errorf("the refused advance's next step = %s, want open/ (nothing claimed)", got) + } +} + +// The ROLE has no anchor cap and the operation refuses more than 100: the +// client splits, and every anchor comes back once, in request order. +func TestServedEdgeReaderSplitsPastTheWireCap(t *testing.T) { + env := newServedEnv(t, "hedgecap") + ctx := t.Context() + if err := env.createIssue(ctx, &types.Issue{ID: "hedgecap-a", Title: "a", IssueType: types.TypeTask, Status: types.StatusOpen}, "t"); err != nil { + t.Fatal(err) + } + if err := env.createIssue(ctx, &types.Issue{ID: "hedgecap-b", Title: "b", IssueType: types.TypeTask, Status: types.StatusOpen}, "t"); err != nil { + t.Fatal(err) + } + if err := env.addDependency(ctx, &types.Dependency{IssueID: "hedgecap-b", DependsOnID: "hedgecap-a", Type: types.DepBlocks}, "t"); err != nil { + t.Fatal(err) + } + ids := make([]string, 0, 250) + for i := range 248 { + ids = append(ids, fmt.Sprintf("hedgecap-missing-%03d", i)) + } + ids = append(ids, "hedgecap-a", "hedgecap-b") + reader, err := env.subject.EdgeReader() + if err != nil { + t.Fatal(err) + } + res, err := reader.ReadEdges(ctx, issueops.EdgeReadRequest{IDs: ids}) + if err != nil { + t.Fatalf("ReadEdges over %d anchors: %v", len(ids), err) + } + if len(res.Anchors) != len(ids) { + t.Fatalf("ReadEdges answered %d anchors for %d", len(res.Anchors), len(ids)) + } + for i, anchor := range res.Anchors { + if anchor.ID != ids[i] { + t.Fatalf("anchor %d = %s, want %s (request order)", i, anchor.ID, ids[i]) + } + } + last := res.Anchors[len(res.Anchors)-1] + if last.Missing || len(last.Edges) != 1 || last.Edges[0].DependsOnID != "hedgecap-a" { + t.Errorf("hedgecap-b = %+v, want its one edge to hedgecap-a", last) + } + if !res.Anchors[0].Missing { + t.Errorf("an id naming nothing was not reported missing: %+v", res.Anchors[0]) + } +} + +// Against a server without issues.batchGet the role still answers: one +// getIssue per distinct id, the same rows and the same Missing. +func TestServedBatchGetterAnswersAgainstAServerWithoutTheBatch(t *testing.T) { + env := newServedEnv(t, "hbgfb") + ctx := t.Context() + if err := env.createIssue(ctx, &types.Issue{ID: "hbgfb-a", Title: "a", IssueType: types.TypeTask, Status: types.StatusOpen, Labels: []string{"x"}}, "t"); err != nil { + t.Fatal(err) + } + batchGet, _ := wire.CapabilityFor(wire.OpBatchGetIssues) + getter, err := degradedSubject(t, env, batchGet).BatchGetter() + if err != nil { + t.Fatal(err) + } + res, err := getter.GetMany(ctx, issueops.GetManyRequest{IDs: []string{"hbgfb-a", "hbgfb-nope", "hbgfb-a"}}) + if err != nil { + t.Fatalf("GetMany without issues.batchGet: %v", err) + } + if len(res.Issues) != 1 || res.Issues[0].ID != "hbgfb-a" || !slices.Equal(res.Issues[0].Labels, []string{"x"}) || res.Issues[0].RowVersion == 0 { + t.Errorf("Issues = %+v, want hbgfb-a once, labeled, with its revision", res.Issues) + } + if !slices.Equal(res.Missing, []string{"hbgfb-nope"}) { + t.Errorf("Missing = %v, want [hbgfb-nope]", res.Missing) + } +} diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 01fbc35cc..3c72a4a97 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -732,4 +732,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (143 of 200 methods generated; 57 left to this package's hand-written set). +// NOTE: partial shell (143 of 201 methods generated; 58 left to this package's hand-written set). diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index fe2918170..fa5990c7a 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -107,6 +107,9 @@ var opCapability = map[string]string{ // with a typed capability error instead of dialing a path the older server // would answer as a claim of an issue called ":reclaim". OpReclaimIssues: "issues.reclaim", + // The advance-a-molecule role (molecule.go) dials it when served and + // refuses before dialing when not; it never composes a client-side copy. + OpAdvanceMolecule: "issues.advanceMolecule", } // CapProjectEnforce is the behavior capability the server advertises to announce @@ -230,6 +233,16 @@ const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" // issueops.DefaultCreatePriority explicitly (pinCreateDefaultPriority). const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" +// CapCloseAutoCloseMolecule is the behavior capability announcing that +// closeIssue and batchCloseIssues accept `auto_close_molecule` and answer +// the auto-closed root, spelled exactly as httpapi's +// CapIssuesCloseAutoCloseMolecule. lifecycle.go and batchcloser.go send the +// member only when this is advertised; against an older server a close that +// asks for it is refused before it is dialed with the typed capability +// error — never a silent drop, and never a second, client-side copy of the +// rule. +const CapCloseAutoCloseMolecule = "issues.close.autoCloseMolecule" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -369,6 +382,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, + CapCloseAutoCloseMolecule, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 94a90b83a..08861e232 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -25,6 +25,7 @@ const ( OpGetIssue = "getIssue" OpClaimIssue = "claimIssue" OpCloseIssue = "closeIssue" + OpAdvanceMolecule = "advanceMolecule" OpReopenIssue = "reopenIssue" OpUpdateIssue = "updateIssue" OpSweepIssues = "sweepIssues" @@ -98,11 +99,12 @@ const ( // its colon must be literal — which is the whole reason escapeSegment escapes a // colon inside the id itself. const ( - MethodClaim = ":claim" - MethodClose = ":close" - MethodReopen = ":reopen" - MethodRelease = ":release" - MethodCASMetadata = ":casMetadata" + MethodClaim = ":claim" + MethodClose = ":close" + MethodReopen = ":reopen" + MethodAdvanceMolecule = ":advanceMolecule" + MethodRelease = ":release" + MethodCASMetadata = ":casMetadata" ) // The sub-resource collections that hang off the issue-detail path. Unlike a @@ -128,7 +130,7 @@ func IssuePath(id string) (string, error) { // final segment: /v0/beads/issues/{id}:claim and its two siblings. func IssueMethodPath(id, method string) (string, error) { switch method { - case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata: + case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata, MethodAdvanceMolecule: default: return "", fmt.Errorf("no custom method %q on the issue resource", method) } diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index fe2fc7ace..ec690e114 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -110,17 +110,18 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) } // The dimension is pinned so the matrix cannot shrink silently: the first - // slice served six operations and the tip surface has forty-three, so - // thirty-seven of them are things a first-slice server has never heard of. The + // slice served six operations and the tip surface has forty-four, so + // thirty-eight of them are things a first-slice server has never heard of. The // wire wave moved this from twenty-seven to thirty — claimNext, release and // count — wave 2 to thirty-two, adding the dependency-edge count and the // related read, the mini-sync to thirty-five: the comment write and the two // config writes, the batch read (upstream #7248) to thirty-six, and the // stale-lease sweep to thirty-seven. Each is // an operation a first-slice server never served and this client does not - // yet dial, which is exactly the shape the refusal matrix is about. - if len(refused) != 37 { - t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 37", len(refused)) + // yet dial, which is exactly the shape the refusal matrix is about. The + // molecule advance (S16) makes thirty-eight. + if len(refused) != 38 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 38", len(refused)) } // The compat op forces the handshake yet preflights clean: issues.claim is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index b1eb0e6a1..ea4d4d21d 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -69,6 +69,7 @@ func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { {OpWatchEvents, httpapi.OpWatchEvents}, {OpBatchGetIssues, httpapi.OpBatchGetIssues}, {OpReclaimIssues, httpapi.OpReclaimIssues}, + {OpAdvanceMolecule, httpapi.OpAdvanceMolecule}, } // The table is a HAND LIST, and a hand list's failure mode is omission: a new @@ -200,10 +201,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, + httpapi.CapIssuesCloseAutoCloseMolecule, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage and issues.update.allowTemplate and issues.create.defaultPriority tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority and issues.close.autoCloseMolecule tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 33c2fb62c..0532247bb 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -192,6 +192,13 @@ func writeShapes() []writeShape { // (#5506) and the client wave sends it (ga-jbuyf), so it left // the pending set for this one. "ExpectedVersion": member("expected_version"), + // Sent where issues.close.autoCloseMolecule is advertised. + // Against a server that does not advertise it, a close that + // asks for it is refused before dialing (the typed + // storage.ErrUnsupported naming the token), and bd close + // sends the close without it and prints the unserved + // notice instead (S16). + "AutoCloseMolecule": member("auto_close_molecule"), }, }, { @@ -456,10 +463,11 @@ func writeShapes() []writeShape { source: reflect.TypeOf(issueops.CloseBatchRequest{}), body: reflect.TypeOf(apigen.BatchCloseRequest{}), carried: map[string]carriage{ - "Actor": member("actor"), - "Items": member("items"), - "Session": member("session"), - "Force": member("force"), + "Actor": member("actor"), + "Items": member("items"), + "Session": member("session"), + "Force": member("force"), + "AutoCloseMolecule": member("auto_close_molecule"), }, }, { diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index e4a8b5a27..c3390f6ac 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -41,6 +41,7 @@ go_library( "hook_lease_reclaimer.go", "hook_memories.go", "hook_metadata_cas.go", + "hook_molecule_stepper.go", "hook_querier.go", "hook_ready_claimer.go", "hook_ready_counter.go", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 1f79aa088..2be809cc9 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -52,6 +52,7 @@ go_library( "merge_slot.go", "metadata_cas.go", "metadata_schema.go", + "molecule_stepper.go", "open.go", "provenance.go", "pull_report.go", @@ -285,6 +286,7 @@ go_test( "metadata_validation_test.go", "migration_0052_test.go", "mixed_table_lifecycle_test.go", + "molecule_contract_test.go", "multistore_multiprocess_test.go", "open_test.go", "pool_timeouts_test.go", diff --git a/internal/storage/dolt/molecule_contract_test.go b/internal/storage/dolt/molecule_contract_test.go new file mode 100644 index 000000000..b37f5895b --- /dev/null +++ b/internal/storage/dolt/molecule_contract_test.go @@ -0,0 +1,266 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "sync" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestMoleculeContract runs the molecule contract (in-transaction auto-close on +// Lifecycle.Close and BatchCloser.CloseBatch, and the MoleculeStepper) against +// the server-backed store. One store, sequential subtests, ids namespaced by +// the fixture prefix. +func TestMoleculeContract(t *testing.T) { + fixture, ctx, cleanup := newDoltMoleculeFixture(t, "mol") + defer cleanup() + + for _, test := range []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } { + t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) }) + } +} + +func newDoltMoleculeFixture(t *testing.T, prefix string) (conformance.MoleculeFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + cleanup := func() { + cancel() + storeCleanup() + } + lifecycle, err := store.IssueLifecycle() + if err != nil { + cleanup() + t.Fatalf("IssueLifecycle(): %v", err) + } + closer, err := store.BatchCloser() + if err != nil { + cleanup() + t.Fatalf("BatchCloser(): %v", err) + } + stepper, err := store.MoleculeStepper() + if err != nil { + cleanup() + t.Fatalf("MoleculeStepper(): %v", err) + } + kit := newDoltRoleFixtureKit(store, prefix) + return conformance.MoleculeFixture{ + IssuePrefix: kit.IssuePrefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + }, ctx, cleanup +} + +// seedRacingMolecule seeds an auto-closing molecule root with n open steps +// and answers the steps. +func seedRacingMolecule(t *testing.T, ctx context.Context, store *DoltStore, root string, n int) []string { + t.Helper() + if err := store.CreateIssue(ctx, &types.Issue{ID: root, Title: "molecule " + root, IssueType: types.TypeMolecule, Status: types.StatusOpen, Priority: 2}, "tester"); err != nil { + t.Fatalf("create %s: %v", root, err) + } + steps := make([]string, n) + for i := range steps { + steps[i] = fmt.Sprintf("%s.%d", root, i+1) + createPerm(t, ctx, store, steps[i]) + addDependency(t, ctx, store, steps[i], root, types.DepParentChild) + } + return steps +} + +// readStatus reads id's status from the issues table. +func readStatus(t *testing.T, ctx context.Context, store *DoltStore, id string) string { + t.Helper() + var status string + if err := store.db.QueryRowContext(ctx, "SELECT status FROM issues WHERE id = ?", id).Scan(&status); err != nil { + t.Fatalf("read %s status: %v", id, err) + } + return status +} + +// closeTwoStepsInRacingTransactions closes steps[0] and steps[1], each with +// the molecule auto-close, in two transactions that both begin before either +// closes, so each reads the other's step open: the race's interleaving, made +// deterministic. Neither may auto-close the root. It commits both and answers +// the two commit errors. +func closeTwoStepsInRacingTransactions(t *testing.T, ctx context.Context, store *DoltStore, steps []string) [2]error { + t.Helper() + var txs [2]*sql.Tx + for i := range txs { + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin transaction %d: %v", i, err) + } + defer tx.Rollback() // no-op after Commit; preserves cleanup on early Fatal. + txs[i] = tx + } + for i, tx := range txs { + res, _, err := issueops.ExecuteClose(ctx, tx, publicops.CloseRequest{ + Actor: "tester", IssueID: steps[i], Reason: "done", AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + if res.AutoClosedMolecule != nil { + t.Fatalf("close %s auto-closed %s, but in its own transaction %s is still open", steps[i], res.AutoClosedMolecule.ID, steps[1-i]) + } + } + return [2]error{txs[0].Commit(), txs[1].Commit()} +} + +// TestMoleculeAutoCloseLastStepsCannotBothMissTheRoot is the controlled +// two-transaction regression for CloseRequest.AutoCloseMolecule. The closes of +// a molecule's last two open steps, in concurrent transactions, each read the +// other step as still open, so neither closes the root, and their own writes +// touch disjoint rows. If both commit, every step is closed and the root is +// stranded open. The coordination cell both rewrite +// (issueops.TouchMoleculeCoordinationInTx) must make one of them conflict +// instead, so that its retry reads the other's close; the test below drives +// that retry through the store. +func TestMoleculeAutoCloseLastStepsCannotBothMissTheRoot(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + const root = "mol-race" + commitErrs := closeTwoStepsInRacingTransactions(t, ctx, store, seedRacingMolecule(t, ctx, store, root, 2)) + t.Logf("last-step commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatalf("stranded molecule: both last-step closes committed and %s is %s with every step closed; one transaction must conflict", + root, readStatus(t, ctx, store, root)) + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStep pins the cost +// the coordination cell's docs state (issueops.TouchMoleculeCoordinationInTx, +// CloseRequest.AutoCloseMolecule, the CHANGELOG): every close under an open +// auto-closing root rewrites the cell, so two concurrent sibling closes +// conflict even while a third step keeps the molecule from completing. If this +// stops conflicting, those docs are stale. +func TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStep(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + commitErrs := closeTwoStepsInRacingTransactions(t, ctx, store, seedRacingMolecule(t, ctx, store, "mol-siblings", 3)) + t.Logf("sibling commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatal("two concurrent sibling closes under an open auto-closing root both committed; want one serialization conflict") + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// raceMoleculeCloses seeds an auto-closing molecule of n steps under root and +// closes every step at once through lifecycle, whose retry replays each close +// that loses the coordination conflict. Every close must land, and the root +// must end closed, reported by exactly one of them. +func raceMoleculeCloses(t *testing.T, ctx context.Context, store *DoltStore, lifecycle publicops.Lifecycle, root string, n int) { + t.Helper() + steps := seedRacingMolecule(t, ctx, store, root, n) + var ( + start = make(chan struct{}) + wg sync.WaitGroup + results = make([]publicops.CloseResult, n) + errs = make([]error, n) + ) + for i, step := range steps { + wg.Add(1) + go func() { + defer wg.Done() + <-start + results[i], errs[i] = lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "tester", IssueID: step, Reason: "done", AutoCloseMolecule: true, + }) + }() + } + close(start) + wg.Wait() + + reported := 0 + for i, err := range errs { + if err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + if results[i].AutoClosedMolecule != nil { + reported++ + } + } + if status := readStatus(t, ctx, store, root); status != string(types.StatusClosed) { + t.Fatalf("%s is %s after all %d of its steps closed, want closed", root, status, n) + } + if reported != 1 { + t.Errorf("%s was reported by %d closes, want exactly one", root, reported) + } +} + +// TestMoleculeAutoCloseConcurrentLastStepsCloseTheRoot races the closes of a +// molecule's last two steps through the store's Lifecycle role, whose retry +// replays the transaction that loses the coordination conflict: in every +// round the root ends closed, reported by exactly one of the two closes. +func TestMoleculeAutoCloseConcurrentLastStepsCloseTheRoot(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + lifecycle, err := store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const rounds = 8 + for round := range rounds { + raceMoleculeCloses(t, ctx, store, lifecycle, fmt.Sprintf("mol-race-%d", round), 2) + } +} + +// TestMoleculeAutoCloseSiblingFanOutClosesTheRoot closes all the steps of one +// molecule at once. They serialize on the root's coordination cell, and the +// store's retry carries every one of them to a commit: the fan-out costs +// retries, not closes, and the root is closed exactly once. +func TestMoleculeAutoCloseSiblingFanOutClosesTheRoot(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + lifecycle, err := store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + raceMoleculeCloses(t, ctx, store, lifecycle, "mol-fanout", 6) +} diff --git a/internal/storage/dolt/molecule_stepper.go b/internal/storage/dolt/molecule_stepper.go new file mode 100644 index 000000000..537e517e2 --- /dev/null +++ b/internal/storage/dolt/molecule_stepper.go @@ -0,0 +1,17 @@ +package dolt + +import ( + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the advance-a-molecule surface for this store: +// issueops.NewMoleculeStepper composed over this store's own BatchGetter, +// Relations, EdgeReader, Claimer and Lifecycle, so the advance reads and +// claims exactly as those roles do and no second copy of the rule exists. +func (s *DoltStore) MoleculeStepper() (issueops.MoleculeStepper, error) { + roles, err := issueops.MoleculeStepperRolesOf(s) + if err != nil { + return nil, err + } + return issueops.NewMoleculeStepper(roles), nil +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 474d6cc45..88f67c200 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -42,6 +42,7 @@ go_library( "memories.go", "merge_slot.go", "metadata_cas.go", + "molecule_stepper.go", "open.go", "open_stub.go", "querier.go", @@ -193,6 +194,7 @@ go_test( "migrate_selfheal_test.go", "migrate_temp_intermediates_test.go", "migrate_wisp_dep_forward_repair_test.go", + "molecule_contract_test.go", "new_methods_test.go", "open_test.go", "post_sql_commit_test.go", diff --git a/internal/storage/embeddeddolt/molecule_contract_test.go b/internal/storage/embeddeddolt/molecule_contract_test.go new file mode 100644 index 000000000..a268b890b --- /dev/null +++ b/internal/storage/embeddeddolt/molecule_contract_test.go @@ -0,0 +1,63 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestMoleculeContract runs the molecule contract against the embedded store. +func TestMoleculeContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "mol") + ctx := t.Context() + fixture := newEmbeddedMoleculeFixture(t, te, "mol") + + for _, test := range []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } { + t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) }) + } +} + +func newEmbeddedMoleculeFixture(t *testing.T, te *testEnv, prefix string) conformance.MoleculeFixture { + t.Helper() + lifecycle, err := te.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + closer, err := te.store.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + stepper, err := te.store.MoleculeStepper() + if err != nil { + t.Fatalf("MoleculeStepper(): %v", err) + } + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.MoleculeFixture{ + IssuePrefix: kit.IssuePrefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + } +} diff --git a/internal/storage/embeddeddolt/molecule_stepper.go b/internal/storage/embeddeddolt/molecule_stepper.go new file mode 100644 index 000000000..ba81d7f5e --- /dev/null +++ b/internal/storage/embeddeddolt/molecule_stepper.go @@ -0,0 +1,19 @@ +//go:build cgo + +package embeddeddolt + +import ( + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the advance-a-molecule surface for this store: +// issueops.NewMoleculeStepper composed over this store's own BatchGetter, +// Relations, EdgeReader, Claimer and Lifecycle, so the advance reads and +// claims exactly as those roles do and no second copy of the rule exists. +func (s *EmbeddedDoltStore) MoleculeStepper() (issueops.MoleculeStepper, error) { + roles, err := issueops.MoleculeStepperRolesOf(s) + if err != nil { + return nil, err + } + return issueops.NewMoleculeStepper(roles), nil +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 619f4d86e..596ca2cc1 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -88,6 +88,12 @@ func (p *uowProvider) BlockingAnnotator() (publicops.BlockingAnnotator, error) { func (p *uowProvider) TreeWalker() (publicops.TreeWalker, error) { return uow.NewTreeWalker(p) } func (p *uowProvider) BatchGetter() (publicops.BatchGetter, error) { return uow.NewBatchGetter(p) } +// MoleculeStepper composes over this decorator's own roles, so the advance's +// claim answers to the external-dependency policy the claimer applies. +func (p *uowProvider) MoleculeStepper() (publicops.MoleculeStepper, error) { + return uow.NewMoleculeStepper(p) +} + func (p *uowProvider) GraphCounter() (publicops.GraphCounter, error) { return uow.NewGraphCounter(p) } @@ -188,6 +194,7 @@ var ( _ uow.IssueLifecycleSource = (*uowProvider)(nil) _ uow.IssueReaderSource = (*uowProvider)(nil) _ uow.IssueClaimerSource = (*uowProvider)(nil) + _ uow.MoleculeStepperSource = (*uowProvider)(nil) _ uow.RelationsSource = (*uowProvider)(nil) _ uow.EdgeReaderSource = (*uowProvider)(nil) _ uow.BlockingAnnotatorSource = (*uowProvider)(nil) diff --git a/internal/storage/hook_batch_closer.go b/internal/storage/hook_batch_closer.go index 730aad25a..a669f0e68 100644 --- a/internal/storage/hook_batch_closer.go +++ b/internal/storage/hook_batch_closer.go @@ -67,6 +67,9 @@ func (o *hookBatchCloser) CloseBatch(ctx context.Context, request issueops.Close if outcome.Changed { o.hooks.CompleteIssueOperationClose(outcome.Issue) } + if outcome.AutoClosedMolecule != nil { + o.hooks.CompleteIssueOperationClose(outcome.AutoClosedMolecule) + } } if result.ClaimedNext != nil { o.hooks.CompleteIssueOperationUpdate(result.ClaimedNext.Issue) diff --git a/internal/storage/hook_issue_operations.go b/internal/storage/hook_issue_operations.go index 7926a5694..2b963ebbe 100644 --- a/internal/storage/hook_issue_operations.go +++ b/internal/storage/hook_issue_operations.go @@ -70,6 +70,11 @@ func (o *hookIssueOperations) Close(ctx context.Context, request issueops.CloseR result, err := o.inner.Close(ctx, request) if err == nil { o.hooks.CompleteIssueOperationClose(result.Issue) + // A molecule root the close also closed in its transaction is a + // close of its own, and announces as one. + if result.AutoClosedMolecule != nil { + o.hooks.CompleteIssueOperationClose(result.AutoClosedMolecule) + } } return result, err } diff --git a/internal/storage/hook_molecule_stepper.go b/internal/storage/hook_molecule_stepper.go new file mode 100644 index 000000000..4c85daedb --- /dev/null +++ b/internal/storage/hook_molecule_stepper.go @@ -0,0 +1,31 @@ +package storage + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the inner store's advance surface, wrapped so a +// claimed step fires on_update — the event every claim fires. An advance that +// claimed nothing wrote nothing and fires nothing. +func (h *HookFiringStore) MoleculeStepper() (issueops.MoleculeStepper, error) { + inner, err := h.inner.MoleculeStepper() + if err != nil { + return nil, err + } + return &hookMoleculeStepper{inner: inner, hooks: h}, nil +} + +type hookMoleculeStepper struct { + inner issueops.MoleculeStepper + hooks *HookFiringStore +} + +func (s *hookMoleculeStepper) Advance(ctx context.Context, req issueops.AdvanceRequest) (issueops.AdvanceResult, error) { + result, err := s.inner.Advance(ctx, req) + if err == nil && result.Claimed && result.NextStep != nil { + s.hooks.CompleteIssueOperationUpdate(result.NextStep) + } + return result, err +} diff --git a/internal/storage/issueops/close_batch.go b/internal/storage/issueops/close_batch.go index 1a1be3a7b..4d468323e 100644 --- a/internal/storage/issueops/close_batch.go +++ b/internal/storage/issueops/close_batch.go @@ -59,8 +59,19 @@ func ValidateCloseBatchRequest(request publicops.CloseBatchRequest) error { func CloseBatchCommitMessage(result publicops.CloseBatchResult) string { ids := make([]string, 0, len(result.Outcomes)) ephemeral := 0 + var autoClosed []string for _, outcome := range result.Outcomes { - if outcome.Err != nil || !outcome.Changed { + if outcome.Err != nil { + continue + } + if root := outcome.AutoClosedMolecule; root != nil { + if IsWisp(root) { + ephemeral++ + } else { + autoClosed = append(autoClosed, root.ID) + } + } + if !outcome.Changed { continue } if closeOutcomeIsEphemeral(outcome) { @@ -79,6 +90,9 @@ func CloseBatchCommitMessage(result publicops.CloseBatchResult) string { case ephemeral > 1: parts = append(parts, fmt.Sprintf("close %d ephemeral items", ephemeral)) } + if len(autoClosed) > 0 { + parts = append(parts, "auto-close "+strings.Join(autoClosed, ", ")) + } if result.ClaimedNext != nil { parts = append(parts, "claim "+result.ClaimedNext.ID) } @@ -162,6 +176,25 @@ func ExecuteCloseBatchWithPolicy(ctx context.Context, tx *sql.Tx, request public } } + if request.AutoCloseMolecule { + closedAt := make([]int, 0, len(result.Outcomes)) + stepIDs := make([]string, 0, len(result.Outcomes)) + for i, outcome := range result.Outcomes { + if outcome.Err == nil { + closedAt = append(closedAt, i) + stepIDs = append(stepIDs, outcome.IssueID) + } + } + autos, autoTables, err := CloseCompletedMoleculesInTx(ctx, tx, stepIDs, request.Actor, request.Session) + if err != nil { + return publicops.CloseBatchResult{}, nil, err + } + tables.Merge(autoTables) + for j, i := range closedAt { + result.Outcomes[i].AutoClosedMolecule, result.Outcomes[i].MoleculeAutoCloseRefusal = autos[j].Root, autos[j].Refusal + } + } + if claimFilter != nil && landed > 0 { claim, claimTables, err := ExecuteClaimNext(ctx, tx, request.Actor, policy.FilterClaim(*claimFilter)) if err != nil { diff --git a/internal/storage/issueops/close_guard_test.go b/internal/storage/issueops/close_guard_test.go index 9fa419d5b..83da8f86f 100644 --- a/internal/storage/issueops/close_guard_test.go +++ b/internal/storage/issueops/close_guard_test.go @@ -2,6 +2,7 @@ package issueops import ( "errors" + "fmt" "testing" "github.com/steveyegge/beads/internal/types" @@ -53,3 +54,24 @@ func TestCheckClosable(t *testing.T) { t.Fatalf("held refusal = %#v", err) } } + +// TestRefusedMoleculeAutoCloseReportsTheRefusalsOwnSentence pins +// RefusedMoleculeAutoClose: whatever the layers between a root's refused close +// and the auto-close wrapped around it (the unit of work's use case and +// repository), Refusal is the sentence the refusing guard or policy wrote. +func TestRefusedMoleculeAutoCloseReportsTheRefusalsOwnSentence(t *testing.T) { + for _, refusal := range []error{ + publicops.NewCloseBlockedError("bd-1", []string{"bd-2"}), + &publicops.CloseOpenChildrenError{IssueID: "bd-1", OpenChildren: 2}, + &publicops.TemplateReadOnlyError{IssueID: "bd-1"}, + &publicops.PinnedError{IssueID: "bd-1"}, + &publicops.CloseNotAssigneeError{IssueID: "bd-1", Assignee: "bob", Actor: "alice"}, + } { + wrapped := fmt.Errorf("close bd-1: %w", fmt.Errorf("db: IssueSQLRepository.CloseChecked bd-1: %w", refusal)) + for _, err := range []error{refusal, wrapped} { + if got := RefusedMoleculeAutoClose(err).Refusal; got != refusal.Error() { + t.Errorf("RefusedMoleculeAutoClose(%q).Refusal = %q, want %q", err, got, refusal.Error()) + } + } + } +} diff --git a/internal/storage/issueops/execution.go b/internal/storage/issueops/execution.go index c701fb55f..bdfdeb743 100644 --- a/internal/storage/issueops/execution.go +++ b/internal/storage/issueops/execution.go @@ -336,7 +336,16 @@ func ExecuteClose(ctx context.Context, tx *sql.Tx, request publicops.CloseReques if err != nil { return publicops.CloseResult{}, nil, err } - return publicops.CloseResult{Issue: hydrated, Changed: changed, OpenChildren: closed.OpenChildren}, tables, nil + result := publicops.CloseResult{Issue: hydrated, Changed: changed, OpenChildren: closed.OpenChildren} + if attempt.AutoCloseMolecule { + auto, autoTables, err := CloseCompletedMoleculeInTx(ctx, tx, attempt.IssueID, attempt.Actor, attempt.Session) + if err != nil { + return publicops.CloseResult{}, nil, err + } + tables.Merge(autoTables) + result.AutoClosedMolecule, result.MoleculeAutoCloseRefusal = auto.Root, auto.Refusal + } + return result, tables, nil } // ExecuteReopen applies a guarded reopen in tx and reports durable tables changed. diff --git a/internal/storage/issueops/molecule.go b/internal/storage/issueops/molecule.go index be765c7a7..ac42c4de8 100644 --- a/internal/storage/issueops/molecule.go +++ b/internal/storage/issueops/molecule.go @@ -3,10 +3,13 @@ package issueops import ( "context" "database/sql" + "errors" "fmt" + "strconv" "strings" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // GetMoleculeProgressInTx returns progress stats for a molecule within an @@ -111,3 +114,191 @@ func GetMoleculeProgressInTx(ctx context.Context, tx *sql.Tx, moleculeID string) return stats, nil } + +// TxMoleculeReader binds the molecule rules (issueops.MoleculeReader) to a +// caller's transaction: it runs the SAME BatchGetter, Relations and EdgeReader +// bodies the store-backed roles run, against tx, so a close that decides +// whether it completed a molecule reads the state its own writes produced. +type TxMoleculeReader struct{ Tx DBTX } + +var _ = publicops.MoleculeReaderOver(TxMoleculeReader{}) + +// GetMany is the BatchGetter body (ExecuteGetMany) in the bound transaction. +func (r TxMoleculeReader) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return ExecuteGetMany(ctx, r.Tx, request) +} + +// Related is the Relations body (ExecuteRelated) in the bound transaction. +func (r TxMoleculeReader) Related(ctx context.Context, request publicops.RelatedRequest) ([]*publicops.RelatedIssue, error) { + if err := ValidateRelatedRequest(request); err != nil { + return nil, err + } + return ExecuteRelated(ctx, r.Tx, request) +} + +// ReadEdges is the EdgeReader body (ExecuteEdgeRead) in the bound transaction. +func (r TxMoleculeReader) ReadEdges(ctx context.Context, request publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + if err := ValidateEdgeReadRequest(request); err != nil { + return publicops.EdgeReadResult{}, err + } + return ExecuteEdgeRead(ctx, r.Tx, request) +} + +// MoleculeAutoClose is what one molecule auto-close did. +type MoleculeAutoClose struct { + // Root is the post-close snapshot of the root it closed, or nil. + Root *types.Issue + // Refusal is why a completed root stayed open (its close policy or a + // close guard refused it), or "": the refusal's own sentence + // (RefusedMoleculeAutoClose). + Refusal string +} + +// RefusedMoleculeAutoClose is the MoleculeAutoClose for a root whose close +// failed with err, a refusal (issueops.IsMoleculeAutoCloseRefusal). Its +// Refusal is the sentence the refusing guard or close policy wrote, the same +// on every backend. ExecuteClose returns a refusal as it was minted, while the +// unit of work's use case and repository wrap it on the way out ("close : +// db: IssueSQLRepository.CloseChecked : …"); the sentence already names +// the root, so the wrap would only leak a repository type into an +// operator-facing field (bd close --json's molecule_auto_close_refusal). +func RefusedMoleculeAutoClose(err error) MoleculeAutoClose { + var ( + blocked *publicops.BlockedError + openChildren *publicops.CloseOpenChildrenError + template *publicops.TemplateReadOnlyError + pinned *publicops.PinnedError + notAssignee *publicops.CloseNotAssigneeError + ) + switch { + case errors.As(err, &blocked): + err = blocked + case errors.As(err, &openChildren): + err = openChildren + case errors.As(err, &template): + err = template + case errors.As(err, &pinned): + err = pinned + case errors.As(err, ¬Assignee): + err = notAssignee + } + return MoleculeAutoClose{Refusal: err.Error()} +} + +// moleculeCoordinationKeyPrefix names the local_metadata cells an auto-close +// rewrites (TouchMoleculeCoordinationInTx). +const moleculeCoordinationKeyPrefix = "molecule-coordination/v1/" + +// MoleculeCoordinationKey is the local_metadata cell a molecule auto-close +// rewrites before it decides rootID. It shares the dependency coordination +// cells' sharding (4096 rows): the same root always resolves to the same +// cell, and a hash collision between two roots only adds a safe +// serialization conflict. +func MoleculeCoordinationKey(rootID string) string { + return fmt.Sprintf("%s%03x", moleculeCoordinationKeyPrefix, dependencyCoordinationShard(rootID)) +} + +// TouchMoleculeCoordinationInTx rewrites rootID's molecule coordination cell +// with a fresh value, the dependency coordination idiom +// (TouchDependencyCoordinationTableInTx) applied to a molecule's completion. +// +// Two closes that finish a molecule's last two steps in concurrent +// transactions each read the other's step as still open, so neither closes the +// root, and their writes touch disjoint rows: both commit and the complete +// molecule's root is stranded open. Every auto-close that decides a root +// rewrites this one cell first, so two transactions deciding the same root +// conflict at commit instead; the loser's retry reads the winner's close and +// closes the root. +// +// The cost is that sibling closes serialize. The cell is rewritten before the +// steps are read, by every close under the open root and not only the one that +// completes it, so any two concurrent closes of steps under the same root +// conflict and one of them retries. Under heavy fan-out a close can spend its +// whole retry budget conflicting and then fails with the serialization error. +func TouchMoleculeCoordinationInTx(ctx context.Context, tx DBTX, rootID string) error { + if rootID == "" { + return fmt.Errorf("touch molecule coordination: root ID must not be empty") + } + if _, err := tx.ExecContext(ctx, + "REPLACE INTO local_metadata (`key`, value) VALUES (?, ?)", MoleculeCoordinationKey(rootID), strconv.FormatInt(FreshRowLock(), 10)); err != nil { + return fmt.Errorf("touch molecule coordination for %s: %w", rootID, err) + } + return nil +} + +// CloseCompletedMoleculeInTx is CloseRequest.AutoCloseMolecule's body for the +// store-backed backends: CloseCompletedMoleculesInTx for one step. +func CloseCompletedMoleculeInTx(ctx context.Context, tx *sql.Tx, stepID, actor, session string) (MoleculeAutoClose, ChangedTables, error) { + autos, tables, err := CloseCompletedMoleculesInTx(ctx, tx, []string{stepID}, actor, session) + if err != nil { + return MoleculeAutoClose{}, nil, err + } + return autos[0], tables, nil +} + +// CloseCompletedMoleculesInTx is CloseBatchRequest.AutoCloseMolecule's body +// for the store-backed backends, and through CloseCompletedMoleculeInTx +// CloseRequest.AutoCloseMolecule's. For every auto-closing molecule root a +// close of stepIDs could have completed (issueops.AutoClosingMoleculeRoots, +// read in tx), it rewrites the root's coordination cell +// (TouchMoleculeCoordinationInTx), and when every step under the root is now +// closed (issueops.MoleculeStepsClosed) it closes the root in tx through +// ExecuteClose — issueops.MoleculeRootClose: unforced, guarded on the root +// revision read in the same transaction, recording +// issueops.MoleculeAutoCloseReason and session. +// +// It answers one MoleculeAutoClose per step id, in order. Each root is decided +// once, and what happened to it is reported on the first step under it; every +// other step under that root answers the zero value. +// +// A refusal of the root (issueops.IsMoleculeAutoCloseRefusal) is answered as +// MoleculeAutoClose.Refusal and writes nothing; the caller's own closes stand. +// Any other failure is returned and fails the caller's transaction. +func CloseCompletedMoleculesInTx(ctx context.Context, tx *sql.Tx, stepIDs []string, actor, session string) ([]MoleculeAutoClose, ChangedTables, error) { + reader := publicops.MoleculeReaderOver(TxMoleculeReader{Tx: tx}) + roots, err := publicops.AutoClosingMoleculeRoots(ctx, reader, stepIDs) + if err != nil { + return nil, nil, err + } + autos := make([]MoleculeAutoClose, len(stepIDs)) + tables := ChangedTables{} + decided := make(map[string]bool, len(roots)) + for i, stepID := range stepIDs { + root := roots[stepID] + if root == nil || decided[root.ID] { + continue + } + decided[root.ID] = true + auto, rootTables, err := closeCompletedMoleculeRootInTx(ctx, tx, reader, root, actor, session) + if err != nil { + return nil, nil, err + } + tables.Merge(rootTables) + autos[i] = auto + } + return autos, tables, nil +} + +func closeCompletedMoleculeRootInTx(ctx context.Context, tx *sql.Tx, reader publicops.MoleculeReader, root *types.Issue, actor, session string) (MoleculeAutoClose, ChangedTables, error) { + if err := TouchMoleculeCoordinationInTx(ctx, tx, root.ID); err != nil { + return MoleculeAutoClose{}, nil, err + } + complete, err := publicops.MoleculeStepsClosed(ctx, reader, root.ID) + if err != nil || !complete { + return MoleculeAutoClose{}, nil, err + } + closed, tables, err := ExecuteClose(ctx, tx, publicops.MoleculeRootClose(root, actor, session)) + if publicops.IsMoleculeAutoCloseRefusal(err) { + return RefusedMoleculeAutoClose(err), nil, nil + } + if err != nil { + // Not a refusal, such as a storage error. Returned, it fails the + // caller's transaction, so the step's own close rolls back with it + // (CloseRequest.AutoCloseMolecule). + return MoleculeAutoClose{}, nil, fmt.Errorf("auto-closing molecule %s: %w", root.ID, err) + } + if !closed.Changed { + return MoleculeAutoClose{}, tables, nil + } + return MoleculeAutoClose{Root: closed.Issue}, tables, nil +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 74fe93699..4b6ee01ee 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -388,6 +388,16 @@ type Storage interface { // event Releaser fires. See hook_lease_reclaimer.go. LeaseReclaimer() (issueops.LeaseReclaimer, error) + // MoleculeStepper returns the advance-a-molecule surface for this store: + // find a molecule's next ready step after one closed and, on request, + // claim it — the operation behind `bd close --continue`. Every local + // backend answers issueops.NewMoleculeStepper composed over its own + // roles; a remote backend serves it over its wire. + // + // It is a WRITE role and its hook decorator WRAPS: a claimed step fires + // on_update, as every claim does. + MoleculeStepper() (issueops.MoleculeStepper, error) + // Issue CRUD CreateIssue(ctx context.Context, issue *types.Issue, actor string) error CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error @@ -1082,6 +1092,62 @@ type ExternalDependencyPolicyProber interface { ServerEnforcesExternalDependencyPolicy(ctx context.Context) (bool, error) } +// MoleculeAutoCloseProber is implemented by a DoltStorage whose closes may not +// accept issueops.CloseRequest.AutoCloseMolecule — today exactly +// httpclient.Store, whose server advertises it with +// issues.close.autoCloseMolecule. Every local backend runs the auto-close in +// the close's own transaction and does not implement this. +// +// A server that does not advertise it (an older bd serve, or a server whose +// orchestrator owns the molecule lifecycle, as gc does) is answered false, and +// the caller closes WITHOUT the flag: asking for it there is refused before +// dialing, and the molecule's auto-close is that server's to run. +type MoleculeAutoCloseProber interface { + ServesMoleculeAutoClose(ctx context.Context) (bool, error) +} + +// ServesMoleculeAutoClose reports whether closes through s may set +// issueops.CloseRequest.AutoCloseMolecule (MoleculeAutoCloseProber, looked up +// through every decorator). A store that does not implement the prober runs +// the auto-close itself and is answered true. A probe that fails is answered +// true as well: the close dials the same handshake and reports its failure, +// so nothing is hidden by asking for the auto-close. +func ServesMoleculeAutoClose(ctx context.Context, s DoltStorage) bool { + prober, ok := UnwrapStore(s).(MoleculeAutoCloseProber) + if !ok { + return true + } + served, err := prober.ServesMoleculeAutoClose(ctx) + return err != nil || served +} + +// MoleculeAdvanceProber is implemented by a DoltStorage whose MoleculeStepper +// may not be served — today exactly httpclient.Store, whose server advertises +// it with issues.advanceMolecule. Every local backend composes the stepper +// over its own roles and does not implement this. +type MoleculeAdvanceProber interface { + ServesMoleculeAdvance(ctx context.Context) (bool, error) +} + +// ServesMoleculeAdvance reports whether s's MoleculeStepper can run +// (MoleculeAdvanceProber, looked up through every decorator). A store that +// does not implement the prober runs the advance itself and is answered true. +// A probe that fails returns its error. +// +// `bd close --continue` asks BEFORE it closes: the advance is a second +// operation after the close commits, so refusing it only then would leave +// the step closed and the next step unclaimed. That is also why a failed +// probe is not answered true, as ServesMoleculeAutoClose answers it: the +// close would commit and only the advance, dialing the same handshake, would +// fail — the half-done state the question exists to avoid. +func ServesMoleculeAdvance(ctx context.Context, s DoltStorage) (bool, error) { + prober, ok := UnwrapStore(s).(MoleculeAdvanceProber) + if !ok { + return true, nil + } + return prober.ServesMoleculeAdvance(ctx) +} + // ExcludeIDsUnsupportedStore is implemented by a DoltStorage whose ready-work // reads cannot express types.WorkFilter.ExcludeIDs over their own transport. // Today that is exactly httpclient.Store: the v0 wire's listReadyWork and diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 4eebb70b6..ac268f448 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "maintenance.go", "memories.go", "metadata_cas.go", + "molecule.go", "notifying.go", "pool.go", "querier.go", @@ -141,6 +142,7 @@ go_test( "memories_contract_test.go", "metadata_cas_contract_test.go", "metadata_patch_dolt_test.go", + "molecule_contract_test.go", "notifying_integration_test.go", "notifying_parity_test.go", "notifying_test.go", diff --git a/internal/storage/uow/batch_closer.go b/internal/storage/uow/batch_closer.go index e29707f42..43cfed688 100644 --- a/internal/storage/uow/batch_closer.go +++ b/internal/storage/uow/batch_closer.go @@ -73,6 +73,24 @@ func (o *batchCloser) CloseBatch(ctx context.Context, request publicops.CloseBat } } + if request.AutoCloseMolecule { + closedAt := make([]int, 0, len(result.Outcomes)) + stepIDs := make([]string, 0, len(result.Outcomes)) + for i, outcome := range result.Outcomes { + if outcome.Err == nil { + closedAt = append(closedAt, i) + stepIDs = append(stepIDs, outcome.IssueID) + } + } + autos, err := CloseCompletedMoleculesInUOW(ctx, uw, stepIDs, request.Actor, request.Session) + if err != nil { + return publicops.CloseBatchResult{}, "", err + } + for j, i := range closedAt { + result.Outcomes[i].AutoClosedMolecule, result.Outcomes[i].MoleculeAutoCloseRefusal = autos[j].Root, autos[j].Refusal + } + } + if claimFilter != nil && landed > 0 { claimed, err := ClaimNextInUOW(ctx, uw, request.Actor, *claimFilter) if err != nil { diff --git a/internal/storage/uow/batchgetter.go b/internal/storage/uow/batchgetter.go index c379d2102..adbd54cc1 100644 --- a/internal/storage/uow/batchgetter.go +++ b/internal/storage/uow/batchgetter.go @@ -39,6 +39,11 @@ var _ publicops.BatchGetter = (*batchGetter)(nil) // body (issueops.ExecuteGetMany), so this leg does no pre-check of its own. func (c *batchGetter) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { return RunTxRead(ctx, c.provider, func(ctx context.Context, uw UnitOfWork) (publicops.GetManyResult, error) { - return uw.IssueUseCase().GetMany(ctx, request) + return GetManyInUOW(ctx, uw, request) }) } + +// GetManyInUOW is the BatchGetter body, run inside the caller's unit of work. +func GetManyInUOW(ctx context.Context, uw UnitOfWork, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return uw.IssueUseCase().GetMany(ctx, request) +} diff --git a/internal/storage/uow/edgereader.go b/internal/storage/uow/edgereader.go index bc4e456da..f288f1bd6 100644 --- a/internal/storage/uow/edgereader.go +++ b/internal/storage/uow/edgereader.go @@ -5,6 +5,7 @@ import ( "fmt" storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" publicops "github.com/steveyegge/beads/issueops" ) @@ -51,33 +52,49 @@ func (r *edgeReader) ReadEdges(ctx context.Context, request publicops.EdgeReadRe return publicops.EdgeReadResult{Anchors: []publicops.AnchorEdges{}}, nil } return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) (publicops.EdgeReadResult, error) { - present := make(map[string]struct{}, len(anchors)) - issues, err := uw.IssueUseCase().GetIssuesByIDs(ctx, anchors) - if err != nil { - return publicops.EdgeReadResult{}, err - } - for _, issue := range issues { - if issue != nil { - present[issue.ID] = struct{}{} - } - } - wisps, err := uw.IssueUseCase().GetWispsByIDs(ctx, anchors) - if err != nil { - return publicops.EdgeReadResult{}, err - } - for _, wisp := range wisps { - if wisp != nil { - present[wisp.ID] = struct{}{} - } - } - edges, err := uw.DependencyUseCase().GetIssueDependencyRecords(ctx, anchors) - if err != nil { - return publicops.EdgeReadResult{}, err - } - // The type filter and the order run HERE rather than in the reads - // above, so both implementations narrow and order through one function; - // a filter pushed into one side's query would put the narrowing in SQL - // on one backend and in Go on the other. - return storageissueops.FinishEdgeRead(anchors, present, edges, request.Types), nil + return readEdgesInUOW(ctx, uw, anchors, request.Types) }) } + +// ReadEdgesInUOW is the EdgeReader body, run inside the caller's unit of work. +func ReadEdgesInUOW(ctx context.Context, uw UnitOfWork, request publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + if err := storageissueops.ValidateEdgeReadRequest(request); err != nil { + return publicops.EdgeReadResult{}, err + } + anchors := storageissueops.EdgeReadAnchors(request.IDs) + if len(anchors) == 0 { + return publicops.EdgeReadResult{Anchors: []publicops.AnchorEdges{}}, nil + } + return readEdgesInUOW(ctx, uw, anchors, request.Types) +} + +func readEdgesInUOW(ctx context.Context, uw UnitOfWork, anchors []string, depTypes []types.DependencyType) (publicops.EdgeReadResult, error) { + present := make(map[string]struct{}, len(anchors)) + issues, err := uw.IssueUseCase().GetIssuesByIDs(ctx, anchors) + if err != nil { + return publicops.EdgeReadResult{}, err + } + for _, issue := range issues { + if issue != nil { + present[issue.ID] = struct{}{} + } + } + wisps, err := uw.IssueUseCase().GetWispsByIDs(ctx, anchors) + if err != nil { + return publicops.EdgeReadResult{}, err + } + for _, wisp := range wisps { + if wisp != nil { + present[wisp.ID] = struct{}{} + } + } + edges, err := uw.DependencyUseCase().GetIssueDependencyRecords(ctx, anchors) + if err != nil { + return publicops.EdgeReadResult{}, err + } + // The type filter and the order run HERE rather than in the reads + // above, so both implementations narrow and order through one function; + // a filter pushed into one side's query would put the narrowing in SQL + // on one backend and in Go on the other. + return storageissueops.FinishEdgeRead(anchors, present, edges, depTypes), nil +} diff --git a/internal/storage/uow/issue_operations.go b/internal/storage/uow/issue_operations.go index 55205753a..b2c5debb0 100644 --- a/internal/storage/uow/issue_operations.go +++ b/internal/storage/uow/issue_operations.go @@ -440,8 +440,15 @@ func (o *issueOperations) Close(ctx context.Context, request publicops.CloseRequ if err != nil { return publicops.CloseResult{}, "", err } - return publicops.CloseResult{Issue: hydrated, Changed: !semanticIssueEqual(before, hydrated), OpenChildren: closed.OpenChildren}, - "close issue", nil + result := publicops.CloseResult{Issue: hydrated, Changed: !semanticIssueEqual(before, hydrated), OpenChildren: closed.OpenChildren} + if attempt.AutoCloseMolecule { + auto, err := CloseCompletedMoleculeInUOW(ctx, uw, attempt.IssueID, attempt.Actor, attempt.Session) + if err != nil { + return publicops.CloseResult{}, "", err + } + result.AutoClosedMolecule, result.MoleculeAutoCloseRefusal = auto.Root, auto.Refusal + } + return result, "close issue", nil }) } diff --git a/internal/storage/uow/molecule.go b/internal/storage/uow/molecule.go new file mode 100644 index 000000000..859130e5b --- /dev/null +++ b/internal/storage/uow/molecule.go @@ -0,0 +1,182 @@ +package uow + +import ( + "context" + "fmt" + "strconv" + + "github.com/steveyegge/beads/internal/storage/domain" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" +) + +// MoleculeReaderInUOW binds the molecule rules (issueops.MoleculeReader) to a +// caller's unit of work: the SAME BatchGetter, Relations and EdgeReader bodies +// this backend's roles run (GetManyInUOW, RelatedInUOW, ReadEdgesInUOW), +// against uw, so a close that decides whether it completed a molecule reads +// the state its own writes produced. It is this backend's twin of +// internal/storage/issueops.TxMoleculeReader. +func MoleculeReaderInUOW(uw UnitOfWork) publicops.MoleculeReader { + return publicops.MoleculeReaderOver(uowMoleculeReader{uw: uw}) +} + +type uowMoleculeReader struct{ uw UnitOfWork } + +func (r uowMoleculeReader) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return GetManyInUOW(ctx, r.uw, request) +} + +func (r uowMoleculeReader) Related(ctx context.Context, request publicops.RelatedRequest) ([]*publicops.RelatedIssue, error) { + if err := storageissueops.ValidateRelatedRequest(request); err != nil { + return nil, err + } + return RelatedInUOW(ctx, r.uw, request) +} + +func (r uowMoleculeReader) ReadEdges(ctx context.Context, request publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + return ReadEdgesInUOW(ctx, r.uw, request) +} + +// CloseCompletedMoleculeInUOW is CloseRequest.AutoCloseMolecule's body for +// this backend: CloseCompletedMoleculesInUOW for one step. +func CloseCompletedMoleculeInUOW(ctx context.Context, uw UnitOfWork, stepID, actor, session string) (storageissueops.MoleculeAutoClose, error) { + autos, err := CloseCompletedMoleculesInUOW(ctx, uw, []string{stepID}, actor, session) + if err != nil { + return storageissueops.MoleculeAutoClose{}, err + } + return autos[0], nil +} + +// CloseCompletedMoleculesInUOW is CloseBatchRequest.AutoCloseMolecule's body +// for this backend, the twin of storageissueops.CloseCompletedMoleculesInTx: +// for every auto-closing molecule root a close of stepIDs could have completed +// (issueops.AutoClosingMoleculeRoots read in uw), it rewrites the root's +// coordination cell (storageissueops.MoleculeCoordinationKey, so two units of +// work deciding the same root conflict at commit and the retry reads the +// other's close; the cost, serialized sibling closes, is +// storageissueops.TouchMoleculeCoordinationInTx's), and when every step under +// the root is now closed (issueops.MoleculeStepsClosed) it closes the root in +// uw — unforced, guarded on the root revision read in the same unit of work, +// recording issueops.MoleculeAutoCloseReason and session. +// +// It answers one MoleculeAutoClose per step id, in order: each root is decided +// once and reported on the first step under it. A refusal of the root +// (issueops.IsMoleculeAutoCloseRefusal) is answered as the Refusal and leaves +// the root untouched; the caller's own closes stand. Any other failure is +// returned and fails the caller's unit of work. +func CloseCompletedMoleculesInUOW(ctx context.Context, uw UnitOfWork, stepIDs []string, actor, session string) ([]storageissueops.MoleculeAutoClose, error) { + reader := MoleculeReaderInUOW(uw) + roots, err := publicops.AutoClosingMoleculeRoots(ctx, reader, stepIDs) + if err != nil { + return nil, err + } + autos := make([]storageissueops.MoleculeAutoClose, len(stepIDs)) + decided := make(map[string]bool, len(roots)) + for i, stepID := range stepIDs { + root := roots[stepID] + if root == nil || decided[root.ID] { + continue + } + decided[root.ID] = true + auto, err := closeCompletedMoleculeRootInUOW(ctx, uw, reader, root, actor, session) + if err != nil { + return nil, err + } + autos[i] = auto + } + return autos, nil +} + +func closeCompletedMoleculeRootInUOW(ctx context.Context, uw UnitOfWork, reader publicops.MoleculeReader, root *publicops.Issue, actor, session string) (storageissueops.MoleculeAutoClose, error) { + // storageissueops.TouchMoleculeCoordinationInTx, through this unit of work. + touch := strconv.FormatInt(storageissueops.FreshRowLock(), 10) + if err := uw.ConfigUseCase().SetLocalMetadata(ctx, storageissueops.MoleculeCoordinationKey(root.ID), touch); err != nil { + return storageissueops.MoleculeAutoClose{}, fmt.Errorf("touch molecule coordination for %s: %w", root.ID, err) + } + complete, err := publicops.MoleculeStepsClosed(ctx, reader, root.ID) + if err != nil || !complete { + return storageissueops.MoleculeAutoClose{}, err + } + req := publicops.MoleculeRootClose(root, actor, session) + // A probe, not a write: an UpdateSpec with no Fields applies nothing. It + // only checks the root's revision against req.ExpectedVersion, the guard + // the close below does not take itself. + if _, err := uw.IssueUseCase().ApplyUpdate(ctx, req.IssueID, domain.UpdateSpec{ExpectedVersion: req.ExpectedVersion}, req.Actor); err != nil { + if publicops.IsMoleculeAutoCloseRefusal(err) { + return storageissueops.RefusedMoleculeAutoClose(err), nil + } + return storageissueops.MoleculeAutoClose{}, fmt.Errorf("auto-closing molecule %s: %w", root.ID, err) + } + params := domain.CloseIssueParams{Reason: req.Reason, Session: req.Session} + var closed domain.CloseIssueResult + if storageissueops.IsWisp(root) { + closed, err = uw.IssueUseCase().CloseWispChecked(ctx, req.IssueID, params, req.Actor, req.Force) + } else { + closed, err = uw.IssueUseCase().CloseIssueChecked(ctx, req.IssueID, params, req.Actor, req.Force) + } + if publicops.IsMoleculeAutoCloseRefusal(err) { + return storageissueops.RefusedMoleculeAutoClose(err), nil + } + if err != nil { + return storageissueops.MoleculeAutoClose{}, fmt.Errorf("auto-closing molecule %s: %w", root.ID, err) + } + if !closed.Closed { + return storageissueops.MoleculeAutoClose{}, nil + } + current := root + if closed.Issue != nil { + current = closed.Issue + } + hydrated, err := hydrateIssueOperation(ctx, uw, current, false, false) + if err != nil { + return storageissueops.MoleculeAutoClose{}, err + } + return storageissueops.MoleculeAutoClose{Root: hydrated}, nil +} + +// MoleculeStepperSource is the capability accessor a unit-of-work provider +// offers for the advance-a-molecule role. +type MoleculeStepperSource interface { + MoleculeStepper() (publicops.MoleculeStepper, error) +} + +// MoleculeStepper returns the advance-a-molecule surface for this provider. +func (p *doltSQLProvider) MoleculeStepper() (publicops.MoleculeStepper, error) { + return NewMoleculeStepper(p) +} + +// NewMoleculeStepper is issueops.NewMoleculeStepper composed over the roles +// THIS provider builds (its batch getter, relations, edge reader, claimer and +// lifecycle), so a wrapper that builds its roles over itself — the notifying +// provider, a policy decorator, the server's timed provider — gets an advance +// whose every read and claim goes through its own layer. +func NewMoleculeStepper(provider UnitOfWorkProvider) (publicops.MoleculeStepper, error) { + if isNilUnitOfWorkProvider(provider) { + return nil, fmt.Errorf("new molecule stepper: unit-of-work provider must not be nil") + } + getter, err := NewBatchGetter(provider) + if err != nil { + return nil, err + } + relations, err := NewIssueRelations(provider) + if err != nil { + return nil, err + } + edges, err := NewEdgeReader(provider) + if err != nil { + return nil, err + } + claimer, err := NewIssueClaimer(provider) + if err != nil { + return nil, err + } + lifecycle, err := NewIssueOperations(provider) + if err != nil { + return nil, err + } + return publicops.NewMoleculeStepper(publicops.MoleculeStepperRoles{ + Reader: publicops.MoleculeReader{BatchGetter: getter, Relations: relations, EdgeReader: edges}, + Claimer: claimer, + Lifecycle: lifecycle, + }), nil +} diff --git a/internal/storage/uow/molecule_contract_test.go b/internal/storage/uow/molecule_contract_test.go new file mode 100644 index 000000000..a4ea4c1ff --- /dev/null +++ b/internal/storage/uow/molecule_contract_test.go @@ -0,0 +1,257 @@ +package uow + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestMoleculeContract runs the molecule contract against the unit-of-work +// backend, whose auto-close body (CloseCompletedMoleculeInUOW) is its own and +// whose stepper is composed over this provider's roles. One provider for the +// suite; sequential subtests namespaced by prefix. +func TestMoleculeContract(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "mol") + fixture := newUOWMoleculeFixture(t, provider, "mol") + + for _, test := range []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } { + t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) }) + } +} + +func newUOWMoleculeFixture(t *testing.T, provider UnitOfWorkProvider, prefix string) conformance.MoleculeFixture { + t.Helper() + lifecycle, err := NewIssueOperations(provider) + if err != nil { + t.Fatalf("NewIssueOperations: %v", err) + } + closer, err := NewBatchCloser(provider) + if err != nil { + t.Fatalf("NewBatchCloser: %v", err) + } + source, ok := provider.(MoleculeStepperSource) + if !ok { + t.Fatalf("provider %T does not offer the MoleculeStepper accessor", provider) + } + stepper, err := source.MoleculeStepper() + if err != nil { + t.Fatalf("MoleculeStepper(): %v", err) + } + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.MoleculeFixture{ + IssuePrefix: kit.IssuePrefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + } +} + +// seedRacingMoleculeInUOW seeds an auto-closing molecule root with n open +// steps through kit and answers the steps. +func seedRacingMoleculeInUOW(t *testing.T, ctx context.Context, kit roleFixtureKit, root string, n int) []string { + t.Helper() + if err := kit.CreateIssue(ctx, &types.Issue{ID: root, Title: "molecule " + root, IssueType: types.TypeMolecule, Status: types.StatusOpen, Priority: 2}, "tester"); err != nil { + t.Fatalf("seed %s: %v", root, err) + } + steps := make([]string, n) + for i := range steps { + steps[i] = fmt.Sprintf("%s.%d", root, i+1) + if err := kit.CreateIssue(ctx, &types.Issue{ID: steps[i], Title: "step " + steps[i], IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "tester"); err != nil { + t.Fatalf("seed %s: %v", steps[i], err) + } + if err := kit.AddDependency(ctx, &types.Dependency{IssueID: steps[i], DependsOnID: root, Type: types.DepParentChild}, "tester"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", steps[i], root, err) + } + } + return steps +} + +func readStatusInUOW(t *testing.T, ctx context.Context, kit roleFixtureKit, id string) string { + t.Helper() + var status string + if err := kit.QueryScalar(ctx, "SELECT status FROM issues WHERE id = ?", []any{id}, &status); err != nil { + t.Fatalf("read %s status: %v", id, err) + } + return status +} + +// closeTwoStepsInRacingUOWs closes steps[0] and steps[1], each followed by +// this backend's auto-close body (CloseCompletedMoleculeInUOW), in two units +// of work that both decide before either commits, so each reads the other's +// step open: the race's interleaving, made deterministic. Neither may +// auto-close the root. It commits both and answers the two commit errors. +func closeTwoStepsInRacingUOWs(t *testing.T, ctx context.Context, provider UnitOfWorkProvider, steps []string) [2]error { + t.Helper() + var uws [2]UnitOfWork + for i := range uws { + uw, err := provider.NewUOW(ctx) + if err != nil { + t.Fatalf("open unit of work %d: %v", i, err) + } + defer uw.Close(ctx) // no-op after Commit; preserves cleanup on early Fatal. + uws[i] = uw + } + for i, uw := range uws { + if _, err := uw.IssueUseCase().CloseIssueChecked(ctx, steps[i], domain.CloseIssueParams{Reason: "done"}, "tester", false); err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + auto, err := CloseCompletedMoleculeInUOW(ctx, uw, steps[i], "tester", "") + if err != nil { + t.Fatalf("auto-close after %s: %v", steps[i], err) + } + if auto.Root != nil { + t.Fatalf("close %s auto-closed %s, but in its own unit of work %s is still open", steps[i], auto.Root.ID, steps[1-i]) + } + } + return [2]error{uws[0].Commit(ctx, "close "+steps[0]), uws[1].Commit(ctx, "close "+steps[1])} +} + +// TestMoleculeAutoCloseLastStepsCannotBothMissTheRootInUOW is the unit-of-work +// twin of the dolt store's two-transaction regression: two units of work that +// each close one of a molecule's last two open steps and run this backend's +// auto-close body (CloseCompletedMoleculeInUOW) each read the other's step +// open, so neither closes the root. The coordination cell both rewrite must +// make one commit conflict, or the complete molecule's root is stranded open. +func TestMoleculeAutoCloseLastStepsCannotBothMissTheRootInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molrace") + kit := newUOWRoleFixtureKit(provider, "molrace") + root := kit.IssuePrefix + "-race" + + commitErrs := closeTwoStepsInRacingUOWs(t, ctx, provider, seedRacingMoleculeInUOW(t, ctx, kit, root, 2)) + t.Logf("last-step commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatalf("stranded molecule: both last-step closes committed and %s is %s with every step closed; one unit of work must conflict", + root, readStatusInUOW(t, ctx, kit, root)) + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStepInUOW is the +// unit-of-work twin of the dolt store's pin on the coordination cell's cost: +// two concurrent sibling closes under an open auto-closing root conflict even +// while a third step keeps the molecule from completing. If this stops +// conflicting, the docs that state the cost are stale. +func TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStepInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molsib") + kit := newUOWRoleFixtureKit(provider, "molsib") + + commitErrs := closeTwoStepsInRacingUOWs(t, ctx, provider, seedRacingMoleculeInUOW(t, ctx, kit, kit.IssuePrefix+"-siblings", 3)) + t.Logf("sibling commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatal("two concurrent sibling closes under an open auto-closing root both committed; want one serialization conflict") + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// raceMoleculeClosesInUOW seeds an auto-closing molecule of n steps under +// root and closes every step at once through lifecycle, whose RunTxResult +// replays each unit of work that loses the coordination conflict. Every close +// must land, and the root must end closed, reported by exactly one of them. +func raceMoleculeClosesInUOW(t *testing.T, ctx context.Context, kit roleFixtureKit, lifecycle publicops.Lifecycle, root string, n int) { + t.Helper() + steps := seedRacingMoleculeInUOW(t, ctx, kit, root, n) + var ( + start = make(chan struct{}) + wg sync.WaitGroup + results = make([]publicops.CloseResult, n) + errs = make([]error, n) + ) + for i, step := range steps { + wg.Add(1) + go func() { + defer wg.Done() + <-start + results[i], errs[i] = lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "tester", IssueID: step, Reason: "done", AutoCloseMolecule: true, + }) + }() + } + close(start) + wg.Wait() + + reported := 0 + for i, err := range errs { + if err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + if results[i].AutoClosedMolecule != nil { + reported++ + } + } + if status := readStatusInUOW(t, ctx, kit, root); status != string(types.StatusClosed) { + t.Fatalf("%s is %s after all %d of its steps closed, want closed", root, status, n) + } + if reported != 1 { + t.Errorf("%s was reported by %d closes, want exactly one", root, reported) + } +} + +// TestMoleculeAutoCloseConcurrentLastStepsCloseTheRootInUOW races the closes +// of a molecule's last two steps through this backend's Lifecycle role, whose +// RunTxResult replays the unit of work that loses the coordination conflict: +// in every round the root ends closed, reported by exactly one of the closes. +func TestMoleculeAutoCloseConcurrentLastStepsCloseTheRootInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molagg") + kit := newUOWRoleFixtureKit(provider, "molagg") + lifecycle, err := NewIssueOperations(provider) + if err != nil { + t.Fatalf("NewIssueOperations: %v", err) + } + + const rounds = 8 + for round := range rounds { + raceMoleculeClosesInUOW(t, ctx, kit, lifecycle, fmt.Sprintf("%s-race-%d", kit.IssuePrefix, round), 2) + } +} + +// TestMoleculeAutoCloseSiblingFanOutClosesTheRootInUOW is the unit-of-work +// twin of the dolt store's fan-out: all the steps of one molecule closed at +// once serialize on the root's coordination cell, and RunTxResult carries +// every one of them to a commit, closing the root exactly once. +func TestMoleculeAutoCloseSiblingFanOutClosesTheRootInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molfan") + kit := newUOWRoleFixtureKit(provider, "molfan") + lifecycle, err := NewIssueOperations(provider) + if err != nil { + t.Fatalf("NewIssueOperations: %v", err) + } + + raceMoleculeClosesInUOW(t, ctx, kit, lifecycle, kit.IssuePrefix+"-fanout", 6) +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index 53da8a3b9..d0bafd710 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -242,6 +242,10 @@ func (p *notifyingProvider) BatchGetter() (publicops.BatchGetter, error) { return NewBatchGetter(p) } +func (p *notifyingProvider) MoleculeStepper() (publicops.MoleculeStepper, error) { + return NewMoleculeStepper(p) +} + func (p *notifyingProvider) Counter() (publicops.Counter, error) { return NewCounter(p) } func (p *notifyingProvider) ReadyCounter() (publicops.ReadyCounter, error) { @@ -409,6 +413,7 @@ var ( _ TreeWalkerSource = (*notifyingProvider)(nil) _ GraphCounterSource = (*notifyingProvider)(nil) _ BatchGetterSource = (*notifyingProvider)(nil) + _ MoleculeStepperSource = (*notifyingProvider)(nil) _ CounterSource = (*notifyingProvider)(nil) _ ReadyCounterSource = (*notifyingProvider)(nil) _ ReadyClaimerSource = (*notifyingProvider)(nil) diff --git a/internal/storage/uow/relations.go b/internal/storage/uow/relations.go index 9db1a0815..8e090dc99 100644 --- a/internal/storage/uow/relations.go +++ b/internal/storage/uow/relations.go @@ -48,29 +48,35 @@ func (r *issueRelations) Related(ctx context.Context, request publicops.RelatedR if err := storageissueops.ValidateRelatedRequest(request); err != nil { return nil, err } + return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) ([]*publicops.RelatedIssue, error) { + return RelatedInUOW(ctx, uw, request) + }) +} + +// RelatedInUOW is the Relations body, run inside the caller's unit of work. +// The request must already be validated. +func RelatedInUOW(ctx context.Context, uw UnitOfWork, request publicops.RelatedRequest) ([]*publicops.RelatedIssue, error) { direction := domain.DepDirectionOut if request.Direction == publicops.RelationIn { direction = domain.DepDirectionIn } - return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) ([]*publicops.RelatedIssue, error) { - issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), request.ID) - if err != nil { - return nil, err - } - filter := domain.DepListFilter{Direction: direction} - var items []*publicops.RelatedIssue - if isWisp { - items, err = uw.DependencyUseCase().ListWispWithIssueMetadata(ctx, issue.ID, filter) - } else { - items, err = uw.DependencyUseCase().ListWithIssueMetadata(ctx, issue.ID, filter) - } - if err != nil { - return nil, err - } - // The filter runs HERE rather than in the use-case call above so both - // implementations narrow and order through one function; passing - // filter.Types down would put the narrowing in the query on one side - // and in Go on the other. - return storageissueops.FinishRelatedPage(items, request.Types), nil - }) + issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), request.ID) + if err != nil { + return nil, err + } + filter := domain.DepListFilter{Direction: direction} + var items []*publicops.RelatedIssue + if isWisp { + items, err = uw.DependencyUseCase().ListWispWithIssueMetadata(ctx, issue.ID, filter) + } else { + items, err = uw.DependencyUseCase().ListWithIssueMetadata(ctx, issue.ID, filter) + } + if err != nil { + return nil, err + } + // The filter runs HERE rather than in the use-case call above so both + // implementations narrow and order through one function; passing + // filter.Types down would put the narrowing in the query on one side + // and in Go on the other. + return storageissueops.FinishRelatedPage(items, request.Types), nil } diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index ffcf6b7d5..e1446ce6c 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "lease_reclaimer.go", "memories.go", "metadata_cas.go", + "molecule_stepper.go", "querier.go", "ready_claimer.go", "ready_counter.go", diff --git a/internal/telemetry/molecule_stepper.go b/internal/telemetry/molecule_stepper.go new file mode 100644 index 000000000..50ec721ac --- /dev/null +++ b/internal/telemetry/molecule_stepper.go @@ -0,0 +1,35 @@ +package telemetry + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the inner store's advance surface wrapped in this +// layer's instrumentation. +func (s *InstrumentedStorage) MoleculeStepper() (issueops.MoleculeStepper, error) { + inner, err := s.Unwrap().MoleculeStepper() + if err != nil { + return nil, err + } + return s.WrapMoleculeStepper(inner), nil +} + +// WrapMoleculeStepper instruments molecule advances with this storage layer's +// existing telemetry meter and tracer. +func (s *InstrumentedStorage) WrapMoleculeStepper(inner issueops.MoleculeStepper) issueops.MoleculeStepper { + return &instrumentedMoleculeStepper{storage: s, inner: inner} +} + +type instrumentedMoleculeStepper struct { + storage *InstrumentedStorage + inner issueops.MoleculeStepper +} + +func (m *instrumentedMoleculeStepper) Advance(ctx context.Context, req issueops.AdvanceRequest) (result issueops.AdvanceResult, err error) { + ctx, span, started := m.storage.op(ctx, "MoleculeStepper.Advance") + result, err = m.inner.Advance(ctx, req) + m.storage.done(ctx, span, started, err) + return result, err +} diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 817b49916..d6905629c 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -24,6 +24,9 @@ go_library( "issueops.go", "leasereclaimer.go", "metadatacas.go", + "molecule.go", + "molecule_active.go", + "moleculestepper.go", "querier.go", "reader.go", "reader_ready_scope.go", @@ -51,10 +54,13 @@ go_test( srcs = [ "claimer_external_test.go", "issueops_external_test.go", + "molecule_internal_test.go", + "molecule_test.go", ], + embed = [":issueops"], deps = [ - ":issueops", "//:beads", + "//internal/types", ], ) diff --git a/issueops/batchcloser.go b/issueops/batchcloser.go index bd2861bb1..a2c907bdf 100644 --- a/issueops/batchcloser.go +++ b/issueops/batchcloser.go @@ -69,6 +69,15 @@ type CloseBatchRequest struct { // rule, the same nil-means-nothing-eligible outcome. Restating them here // is how the two would come apart. ClaimNext *ReadyRequest + // AutoCloseMolecule is CloseRequest.AutoCloseMolecule for every item + // that closed or was already closed. It runs after all the closes and + // before ClaimNext, inside the same transaction, so a batch that closes + // several steps of one molecule decides its root once, after every close: + // the root it closed, or its refusal, is reported on the outcome of the + // first item under that root whose Err is nil. An auto-closed root is a + // landing: it is named in the batch's history entry. It does not earn + // ClaimNext's claim on its own. + AutoCloseMolecule bool } // CloseOutcome is what happened to ONE requested item. @@ -85,6 +94,10 @@ type CloseOutcome struct { // OpenChildren is the number of open children observed by a forced close // of this item. It is reported even for an idempotent re-close. OpenChildren int + // AutoClosedMolecule and MoleculeAutoCloseRefusal are + // CloseResult's, for this item, under CloseBatchRequest.AutoCloseMolecule. + AutoClosedMolecule *Issue + MoleculeAutoCloseRefusal string // Err is this item's refusal, and NEVER aborts the rest of the batch. It // carries the same typed close vocabulary Lifecycle.Close returns — // ErrNotFound, ErrCloseBlocked, *CloseOpenChildrenError, diff --git a/issueops/issueops.go b/issueops/issueops.go index 007290a0b..5373b851b 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -395,6 +395,47 @@ type CloseRequest struct { // a miss refuses with ErrVersionMismatch having written nothing. Force // bypasses close policy and never this guard. ExpectedVersion *int64 + // AutoCloseMolecule closes the issue's molecule root IN THE SAME + // TRANSACTION when this close completed it: the root closes itself when + // complete (AutoClosesWhenComplete), it is open, and every step under it + // is now closed (CompletedMolecule). The root close is unforced, guarded + // on the root revision read inside that transaction, and records + // MoleculeAutoCloseReason and this request's Session. + // + // It runs for an idempotent re-close too: the rule is state-derived, so a + // replayed close of the last step heals a root a crash left open. + // + // IT IS OPT-IN because an orchestrator may run its own molecule + // lifecycle (gc closes molecule roots itself); the zero value never + // touches a root. A root close its close policy refuses (a live blocker + // on the root, open children) or a close guard refuses (a template root, a + // pinned root, a root assigned to someone other than Actor — the root + // close is unforced, so each guard holds as it does for a direct close) + // leaves the root open and is reported on + // CloseResult.MoleculeAutoCloseRefusal (IsMoleculeAutoCloseRefusal); it + // never fails the step close. + // + // ANY OTHER FAILURE OF THE ROOT CLOSE FAILS THE STEP'S CLOSE. A storage + // error is returned and rolls the step's close back with it, so a retry + // re-runs both. (The CLI used to close the root after the step's close had + // committed and only warned when that failed, leaving a complete + // molecule's root open.) A write to the root that races this close is not + // such a failure: the two conflict at commit, and the backend's retry + // decides the root again (MoleculeRootClose). + // + // Two closes that finish a molecule's last steps concurrently do not + // strand its root: each close that decides a root first rewrites one + // coordination cell for it, so their transactions conflict at commit and + // the one that retries reads the other's close and closes the root. + // + // THE CELL SERIALIZES SIBLING CLOSES. Every close under an open + // auto-closing root rewrites it, whether or not that close completes the + // molecule, so any two concurrent closes of steps under the same root + // conflict and one of them retries. Under heavy fan-out a close can spend + // its whole retry budget conflicting and then fails with the + // serialization error. Closes under different roots contend only when + // their roots hash to the same cell. + AutoCloseMolecule bool } // ReopenRequest describes an issue reopening. @@ -457,6 +498,15 @@ type CloseResult struct { // OpenChildren is the number of open children observed by a forced close. // It is reported even for an idempotent re-close. OpenChildren int + // AutoClosedMolecule is the molecule root CloseRequest.AutoCloseMolecule + // closed in this close's transaction, as a post-close snapshot with + // labels. Nil when nothing was auto-closed. + AutoClosedMolecule *Issue + // MoleculeAutoCloseRefusal says why a completed molecule's root stayed + // open: its close policy or a close guard refused it + // (IsMoleculeAutoCloseRefusal), in that refusal's own message, the same on + // every backend. Empty otherwise. + MoleculeAutoCloseRefusal string } // ReopenResult reports the post-reopen issue as a detached post-state snapshot diff --git a/issueops/molecule.go b/issueops/molecule.go new file mode 100644 index 000000000..1b787b748 --- /dev/null +++ b/issueops/molecule.go @@ -0,0 +1,869 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/types" +) + +// MoleculeTemplateLabel is the label that marks an epic as a template-driven +// molecule root. `bd mol distill` writes it; a poured molecule's root is +// TypeMolecule instead and carries no label. +const MoleculeTemplateLabel = "template" + +// MoleculeReader is the read surface every molecule operation in this file is +// composed over: three roles every backend serves. +// +// - BatchGetter answers the molecule root and the parent walk's candidate +// roots, labeled, in either plane. +// - Relations answers a node's inbound parent-child steps as FULL rows, in +// either plane. +// - EdgeReader answers the parent walk (a step's outgoing parent-child edge) +// and the blocking edges between a molecule's steps. +// +// THE MOLECULE RULES HAVE ONE IMPLEMENTATION, and it is the functions below. +// A caller that reads committed state binds them to the roles +// (NewMoleculeReader). A close that must decide inside its own +// transaction binds them to the same three role BODIES running against that +// transaction (internal/storage/issueops.TxMoleculeReader, and the unit-of-work +// backend's MoleculeReaderInUOW), so a molecule's membership, completion and +// readiness mean one thing on every backend and every route. +// +// Membership is the parent-child EDGE. A child whose id merely looks +// hierarchical (.) but carries no edge is not a step: no role can +// express an id-pattern scan, and a rule only some routes could apply would +// make the same molecule answer differently per route. +type MoleculeReader struct { + BatchGetter + Relations + EdgeReader +} + +// MoleculeReaderOver binds one value that answers all three role reads — an +// in-transaction binding, a test double — as a MoleculeReader. +func MoleculeReaderOver(r interface { + BatchGetter + Relations + EdgeReader +}) MoleculeReader { + return MoleculeReader{BatchGetter: r, Relations: r, EdgeReader: r} +} + +func (r MoleculeReader) usable() bool { + return r.BatchGetter != nil && r.Relations != nil && r.EdgeReader != nil +} + +// moleculeRoleSource is the accessor set a store or a unit-of-work provider +// offers for the three roles a MoleculeReader composes. +type moleculeRoleSource interface { + BatchGetter() (BatchGetter, error) + IssueRelations() (Relations, error) + EdgeReader() (EdgeReader, error) +} + +// NewMoleculeReader binds a MoleculeReader to src's own role accessors, so +// every decorator src wears (hooks, telemetry, policy) stays in the path. src +// is a storage.DoltStorage or a unit-of-work provider. +func NewMoleculeReader(src moleculeRoleSource) (MoleculeReader, error) { + if src == nil { + return MoleculeReader{}, fmt.Errorf("%w: molecule reader needs a role source", ErrValidation) + } + getter, err := src.BatchGetter() + if err != nil { + return MoleculeReader{}, err + } + relations, err := src.IssueRelations() + if err != nil { + return MoleculeReader{}, err + } + edges, err := src.EdgeReader() + if err != nil { + return MoleculeReader{}, err + } + return MoleculeReader{BatchGetter: getter, Relations: relations, EdgeReader: edges}, nil +} + +// IsMoleculeRoot reports whether issue is the root of a molecule: an epic, a +// TypeMolecule root (`bd mol pour`), or anything carrying the template label. +func IsMoleculeRoot(issue *Issue) bool { + if issue == nil { + return false + } + if issue.IssueType == types.TypeEpic || issue.IssueType == types.TypeMolecule { + return true + } + return hasLabel(issue, MoleculeTemplateLabel) +} + +// AutoClosesWhenComplete reports whether a molecule root closes itself when +// its last step closes. Molecule-typed and ephemeral roots do, and so does a +// template-driven epic; an ordinary epic stays open so it becomes explicit, +// close-eligible work instead of closing as a side effect. +func AutoClosesWhenComplete(root *Issue) bool { + if root == nil { + return false + } + if root.IssueType == types.TypeMolecule || root.Ephemeral { + return true + } + return root.IssueType == types.TypeEpic && hasLabel(root, MoleculeTemplateLabel) +} + +func hasLabel(issue *Issue, label string) bool { + for _, l := range issue.Labels { + if l == label { + return true + } + } + return false +} + +// maxMoleculeDepth bounds the parent walk, so a parent-child cycle ends. +const maxMoleculeDepth = 50 + +// MoleculeRoots walks parent-child edges up from every id and answers, for +// each id that belongs to a molecule, the id of its molecule root. An id whose +// topmost ancestor is not a molecule root (IsMoleculeRoot) is omitted, and so +// is an id that names nothing. A root that is itself a molecule root answers +// itself. +// +// The walk is level by level, one edge read per level for every chain at +// once, so its cost is the depth of the deepest chain, not the number of ids. +// A read the backend fails or refuses is RETURNED: answering "not in a +// molecule" for a read that never happened is how a molecule root gets +// stranded open. +func MoleculeRoots(ctx context.Context, r MoleculeReader, ids []string) (map[string]string, error) { + rows, err := moleculeRootRows(ctx, r, ids) + if err != nil { + return nil, err + } + out := make(map[string]string, len(rows)) + for start, root := range rows { + out[start] = root.ID + } + return out, nil +} + +// moleculeRootRows is MoleculeRoots answering each id's molecule root ROW, as +// the walk read it. +func moleculeRootRows(ctx context.Context, r MoleculeReader, ids []string) (map[string]*Issue, error) { + out := map[string]*Issue{} + if len(ids) == 0 { + return out, nil + } + if !r.usable() { + return nil, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + topOf := make(map[string]string, len(ids)) + current := make(map[string]string, len(ids)) + for _, id := range ids { + current[id] = id + } + for depth := 0; depth < maxMoleculeDepth && len(current) > 0; depth++ { + level := distinctValues(current) + parentOf, err := parentChildParents(ctx, r, level) + if err != nil { + return nil, err + } + next := make(map[string]string, len(current)) + for start, at := range current { + if parent, ok := parentOf[at]; ok { + next[start] = parent + } else { + topOf[start] = at + } + } + current = next + } + for start, at := range current { + topOf[start] = at + } + + tops, err := getIssues(ctx, r, distinctValues(topOf)) + if err != nil { + return nil, fmt.Errorf("loading molecule roots: %w", err) + } + for start, top := range topOf { + if root := tops[top]; IsMoleculeRoot(root) { + out[start] = root + } + } + return out, nil +} + +// MoleculeRoot is MoleculeRoots for one id: the id's molecule root, or "" when +// it belongs to no molecule. +func MoleculeRoot(ctx context.Context, r MoleculeReader, id string) (string, error) { + roots, err := MoleculeRoots(ctx, r, []string{id}) + if err != nil { + return "", err + } + return roots[id], nil +} + +// parentChildParents answers each id's first outgoing parent-child target. +func parentChildParents(ctx context.Context, r MoleculeReader, ids []string) (map[string]string, error) { + res, err := r.ReadEdges(ctx, EdgeReadRequest{IDs: ids, Types: []types.DependencyType{types.DepParentChild}}) + if err != nil { + return nil, fmt.Errorf("walking parent-child edges: %w", err) + } + parentOf := make(map[string]string, len(res.Anchors)) + for _, anchor := range res.Anchors { + for _, edge := range anchor.Edges { + if edge != nil && edge.Type == types.DepParentChild { + parentOf[anchor.ID] = edge.DependsOnID + break + } + } + } + return parentOf, nil +} + +// getIssues reads ids through the BatchGetter in batches the role accepts and +// answers them by id. Missing ids are absent. +func getIssues(ctx context.Context, r MoleculeReader, ids []string) (map[string]*Issue, error) { + out := make(map[string]*Issue, len(ids)) + for start := 0; start < len(ids); start += MaxGetManyIDs { + end := min(start+MaxGetManyIDs, len(ids)) + res, err := r.GetMany(ctx, GetManyRequest{IDs: ids[start:end]}) + if err != nil { + return nil, err + } + for _, issue := range res.Issues { + if issue != nil { + out[issue.ID] = issue + } + } + } + return out, nil +} + +func distinctValues(m map[string]string) []string { + seen := make(map[string]bool, len(m)) + out := make([]string, 0, len(m)) + for _, v := range m { + if !seen[v] { + seen[v] = true + out = append(out, v) + } + } + sort.Strings(out) + return out +} + +// MoleculeGraph is a molecule root, every step under it and the edges between +// them. +type MoleculeGraph struct { + // Root is the molecule root, labeled. + Root *Issue + // Issues holds the root first, then every descendant in depth-first + // pre-order of the parent-child edges. + Issues []*Issue + // Dependencies holds every stored edge whose two ends are both in Issues. + Dependencies []*Dependency + // IssueMap indexes Issues by id. + IssueMap map[string]*Issue +} + +// LoadMolecule reads the molecule rooted at rootID: the root, every +// parent-child descendant as a full row, and the edges between them. A root +// that names nothing is ErrNotFound. A parent-child cycle terminates. +func LoadMolecule(ctx context.Context, r MoleculeReader, rootID string) (*MoleculeGraph, error) { + if !r.usable() { + return nil, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + roots, err := getIssues(ctx, r, []string{rootID}) + if err != nil { + return nil, fmt.Errorf("loading molecule %s: %w", rootID, err) + } + root := roots[rootID] + if root == nil { + return nil, fmt.Errorf("%w: molecule %s", ErrNotFound, rootID) + } + g := &MoleculeGraph{Root: root, Issues: []*Issue{root}, IssueMap: map[string]*Issue{root.ID: root}} + if err := loadMoleculeChildren(ctx, r, g, root.ID); err != nil { + return nil, err + } + + ids := make([]string, len(g.Issues)) + for i, issue := range g.Issues { + ids[i] = issue.ID + } + edges, err := r.ReadEdges(ctx, EdgeReadRequest{IDs: ids}) + if err != nil { + return nil, fmt.Errorf("reading molecule %s edges: %w", rootID, err) + } + for _, anchor := range edges.Anchors { + for _, edge := range anchor.Edges { + if edge == nil { + continue + } + if _, ok := g.IssueMap[edge.DependsOnID]; ok { + g.Dependencies = append(g.Dependencies, edge) + } + } + } + return g, nil +} + +func loadMoleculeChildren(ctx context.Context, r MoleculeReader, g *MoleculeGraph, parentID string) error { + children, err := r.Related(ctx, RelatedRequest{ + ID: parentID, + Direction: RelationIn, + Types: []types.DependencyType{types.DepParentChild}, + }) + if errors.Is(err, ErrNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("reading the steps of %s: %w", parentID, err) + } + for _, child := range children { + if child == nil || child.DependencyType != types.DepParentChild { + continue + } + if _, seen := g.IssueMap[child.ID]; seen { + continue + } + issue := child.Issue + g.Issues = append(g.Issues, &issue) + g.IssueMap[issue.ID] = &issue + if err := loadMoleculeChildren(ctx, r, g, issue.ID); err != nil { + return err + } + } + return nil +} + +// MoleculeStepInfo is one node's readiness inside its molecule. +type MoleculeStepInfo struct { + StepID string `json:"step_id"` + Status string `json:"status"` + IsReady bool `json:"is_ready"` // Can start now (no open blocker) + ParallelGroup string `json:"parallel_group"` // Steps sharing a group can run concurrently + BlockedBy []string `json:"blocked_by"` // Open steps blocking this one + Blocks []string `json:"blocks"` // Steps this one blocks + CanParallel []string `json:"can_parallel"` // Steps that can run in parallel with this one +} + +// MoleculeAnalysis is the readiness and parallelism of every node of a +// molecule, root included. +type MoleculeAnalysis struct { + MoleculeID string `json:"molecule_id"` + TotalSteps int `json:"total_steps"` + ReadySteps int `json:"ready_steps"` + ParallelGroups map[string][]string `json:"parallel_groups"` // group id -> step ids + Steps map[string]*MoleculeStepInfo `json:"steps"` +} + +// AnalyzeMolecule computes readiness from the edges inside the molecule +// rather than from the workspace ready query, which excludes ephemeral rows (a +// wisp molecule's steps are ephemeral by definition). A node is ready when it +// is open or in progress and no open node blocks it through a blocks, +// conditional-blocks or waits-for edge. Nodes at the same blocking depth that +// do not block each other share a parallel group. +func AnalyzeMolecule(g *MoleculeGraph) *MoleculeAnalysis { + analysis := &MoleculeAnalysis{ + MoleculeID: g.Root.ID, + TotalSteps: len(g.Issues), + ParallelGroups: make(map[string][]string), + Steps: make(map[string]*MoleculeStepInfo), + } + + blockedBy := make(map[string]map[string]bool) + blocks := make(map[string]map[string]bool) + parentChildren := make(map[string][]string) + for _, issue := range g.Issues { + blockedBy[issue.ID] = make(map[string]bool) + blocks[issue.ID] = make(map[string]bool) + } + for _, dep := range g.Dependencies { + if dep.Type == types.DepParentChild { + parentChildren[dep.DependsOnID] = append(parentChildren[dep.DependsOnID], dep.IssueID) + } + } + for _, dep := range g.Dependencies { + switch dep.Type { + case types.DepBlocks, types.DepConditionalBlocks: + if _, ok := blockedBy[dep.IssueID]; ok { + blockedBy[dep.IssueID][dep.DependsOnID] = true + } + if _, ok := blocks[dep.DependsOnID]; ok { + blocks[dep.DependsOnID][dep.IssueID] = true + } + case types.DepWaitsFor: + children := parentChildren[dep.DependsOnID] + if len(children) == 0 { + continue + } + if types.ParseWaitsForGateMetadata(dep.Metadata) == types.WaitsForAnyChildren && anyClosed(g, children) { + continue + } + // For all-children (and unresolved any-children) every open child + // blocks the gate. + for _, childID := range children { + child := g.IssueMap[childID] + if child == nil || child.Status == types.StatusClosed { + continue + } + if _, ok := blockedBy[dep.IssueID]; ok { + blockedBy[dep.IssueID][childID] = true + } + if _, ok := blocks[childID]; ok { + blocks[childID][dep.IssueID] = true + } + } + } + } + + for _, issue := range g.Issues { + info := &MoleculeStepInfo{StepID: issue.ID, Status: string(issue.Status), BlockedBy: []string{}, Blocks: []string{}} + for blockerID := range blockedBy[issue.ID] { + if blocker := g.IssueMap[blockerID]; blocker != nil && blocker.Status != types.StatusClosed { + info.BlockedBy = append(info.BlockedBy, blockerID) + } + } + for blockedID := range blocks[issue.ID] { + info.Blocks = append(info.Blocks, blockedID) + } + info.IsReady = (issue.Status == types.StatusOpen || issue.Status == types.StatusInProgress) && len(info.BlockedBy) == 0 + if info.IsReady { + analysis.ReadySteps++ + } + sort.Strings(info.BlockedBy) + sort.Strings(info.Blocks) + analysis.Steps[issue.ID] = info + } + + depths := blockingDepths(g, blockedBy) + depthGroups := make(map[int][]string) + for _, issue := range g.Issues { + depthGroups[depths[issue.ID]] = append(depthGroups[depths[issue.ID]], issue.ID) + } + groupCounter := 0 + for depth := 0; depth <= len(g.Issues); depth++ { + groupCounter = groupParallelSteps(analysis, depthGroups[depth], blocks, blockedBy, groupCounter) + } + return analysis +} + +func anyClosed(g *MoleculeGraph, ids []string) bool { + for _, id := range ids { + if child := g.IssueMap[id]; child != nil && child.Status == types.StatusClosed { + return true + } + } + return false +} + +// groupParallelSteps unions the steps at one blocking depth that do not block +// each other and names every group of two or more. It answers the updated +// group counter. +func groupParallelSteps(analysis *MoleculeAnalysis, steps []string, blocks, blockedBy map[string]map[string]bool, counter int) int { + if len(steps) == 0 { + return counter + } + parent := make(map[string]string, len(steps)) + for _, id := range steps { + parent[id] = id + } + find := func(x string) string { + for parent[x] != x { + parent[x] = parent[parent[x]] + x = parent[x] + } + return x + } + for i, a := range steps { + for _, b := range steps[i+1:] { + if !blocks[a][b] && !blocks[b][a] && !blockedBy[a][b] && !blockedBy[b][a] { + if pa, pb := find(a), find(b); pa != pb { + parent[pa] = pb + } + } + } + } + groups := make(map[string][]string) + var order []string + for _, id := range steps { + root := find(id) + if _, ok := groups[root]; !ok { + order = append(order, root) + } + groups[root] = append(groups[root], id) + } + for _, root := range order { + members := groups[root] + if len(members) < 2 { + continue + } + counter++ + name := fmt.Sprintf("group-%d", counter) + analysis.ParallelGroups[name] = members + for _, id := range members { + info := analysis.Steps[id] + info.ParallelGroup = name + for _, other := range members { + if other != id { + info.CanParallel = append(info.CanParallel, other) + } + } + sort.Strings(info.CanParallel) + } + } + return counter +} + +// blockingDepths answers each node's blocking depth: 0 with no open blocker, +// one more than its deepest open blocker otherwise. A cycle counts as 0. +func blockingDepths(g *MoleculeGraph, blockedBy map[string]map[string]bool) map[string]int { + depths := make(map[string]int) + visiting := make(map[string]bool) + var depthOf func(id string) int + depthOf = func(id string) int { + if d, ok := depths[id]; ok { + return d + } + if visiting[id] { + return 0 + } + visiting[id] = true + deepest := -1 + for blockerID := range blockedBy[id] { + if blocker := g.IssueMap[blockerID]; blocker != nil && blocker.Status != types.StatusClosed { + deepest = max(deepest, depthOf(blockerID)) + } + } + depths[id] = deepest + 1 + return depths[id] + } + for _, issue := range g.Issues { + depthOf(issue.ID) + } + return depths +} + +// Molecule step states, as MoleculeStep.State spells them. +const ( + MoleculeStepDone = "done" + MoleculeStepCurrent = "current" + MoleculeStepReady = "ready" + MoleculeStepBlocked = "blocked" + MoleculeStepPending = "pending" +) + +// MoleculeStep is one step of a MoleculeView. +type MoleculeStep struct { + Issue *Issue + // State is one of the MoleculeStep* constants: done (closed), current (in + // progress), blocked (status blocked), ready (open, no open blocker inside + // the molecule) or pending. + State string + IsCurrent bool +} + +// MoleculeView is where a molecule stands: every step's state, the progress +// counts, the step in progress and the next ready one. +type MoleculeView struct { + Graph *MoleculeGraph + Analysis *MoleculeAnalysis + // Steps holds every step (the root excluded) ordered by how many blocking + // edges inside the molecule each one waits on, fewest first, stable over + // the graph's own order. + Steps []*MoleculeStep + // Completed counts the closed steps and Total every step. + Completed int + Total int + // CurrentStep is the step in progress (the last one in graph order when + // several are); NextStep the first ready step. + CurrentStep *Issue + NextStep *Issue + // Ready holds every node the analysis marks ready (open or in progress + // with no open blocker inside the molecule), root included, in graph + // order: `bd ready --mol`'s list, hydrated rows (assignee and all). + Ready []*Issue +} + +// Complete reports whether every step of the molecule is closed. +func (v *MoleculeView) Complete() bool { return v.Completed >= v.Total } + +// ViewMolecule loads the molecule rooted at rootID and derives its view. +func ViewMolecule(ctx context.Context, r MoleculeReader, rootID string) (*MoleculeView, error) { + g, err := LoadMolecule(ctx, r, rootID) + if err != nil { + return nil, err + } + return MoleculeViewOf(g), nil +} + +// MoleculeViewOf derives the view of an already loaded graph. +func MoleculeViewOf(g *MoleculeGraph) *MoleculeView { + analysis := AnalyzeMolecule(g) + view := &MoleculeView{Graph: g, Analysis: analysis, Total: len(g.Issues) - 1} + for _, issue := range g.Issues { + if info := analysis.Steps[issue.ID]; info != nil && info.IsReady { + view.Ready = append(view.Ready, issue) + } + if issue.ID == g.Root.ID { + continue + } + step := &MoleculeStep{Issue: issue} + switch issue.Status { + case types.StatusClosed: + step.State = MoleculeStepDone + view.Completed++ + case types.StatusInProgress: + step.State = MoleculeStepCurrent + step.IsCurrent = true + view.CurrentStep = issue + case types.StatusBlocked: + step.State = MoleculeStepBlocked + default: + if info := analysis.Steps[issue.ID]; info != nil && info.IsReady { + step.State = MoleculeStepReady + if view.NextStep == nil { + view.NextStep = issue + } + } else { + step.State = MoleculeStepPending + } + } + view.Steps = append(view.Steps, step) + } + sortStepsByBlockers(view.Steps, g) + return view +} + +// ReadySteps answers the steps whose state is ready, in view order. +func (v *MoleculeView) ReadySteps() []*Issue { + var out []*Issue + for _, step := range v.Steps { + if step.State == MoleculeStepReady { + out = append(out, step.Issue) + } + } + return out +} + +func sortStepsByBlockers(steps []*MoleculeStep, g *MoleculeGraph) { + inSteps := make(map[string]bool, len(steps)) + for _, step := range steps { + inSteps[step.Issue.ID] = true + } + waits := make(map[string]int, len(steps)) + for _, dep := range g.Dependencies { + if dep.Type == types.DepBlocks && inSteps[dep.IssueID] && inSteps[dep.DependsOnID] { + waits[dep.IssueID]++ + } + } + sort.SliceStable(steps, func(i, j int) bool { + return waits[steps[i].Issue.ID] < waits[steps[j].Issue.ID] + }) +} + +// MoleculeProgress is a molecule's progress counts over its DIRECT +// parent-child steps. +type MoleculeProgress = types.MoleculeProgressStats + +// ReadMoleculeProgress counts the direct steps of the molecule rooted at +// rootID without loading the whole graph. A root that names nothing is +// ErrNotFound. +func ReadMoleculeProgress(ctx context.Context, r MoleculeReader, rootID string) (*MoleculeProgress, error) { + if !r.usable() { + return nil, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + roots, err := getIssues(ctx, r, []string{rootID}) + if err != nil { + return nil, fmt.Errorf("loading molecule %s: %w", rootID, err) + } + root := roots[rootID] + if root == nil { + return nil, fmt.Errorf("%w: molecule %s", ErrNotFound, rootID) + } + children, err := r.Related(ctx, RelatedRequest{ + ID: rootID, + Direction: RelationIn, + Types: []types.DependencyType{types.DepParentChild}, + }) + if err != nil { + return nil, fmt.Errorf("reading the steps of %s: %w", rootID, err) + } + return MoleculeProgressOf(root, children), nil +} + +// MoleculeProgressOf folds a root's inbound parent-child neighbors into its +// progress counts. CurrentStepID is the first in-progress step in the order +// given. +func MoleculeProgressOf(root *Issue, children []*RelatedIssue) *MoleculeProgress { + stats := &MoleculeProgress{} + if root != nil { + stats.MoleculeID, stats.MoleculeTitle = root.ID, root.Title + } + for _, child := range children { + if child == nil || child.DependencyType != types.DepParentChild { + continue + } + stats.Total++ + switch child.Status { + case types.StatusClosed: + stats.Completed++ + case types.StatusInProgress: + stats.InProgress++ + if stats.CurrentStepID == "" { + stats.CurrentStepID = child.ID + } + } + } + return stats +} + +// CompletedMolecule answers the molecule root that closing stepID completed +// and that closes itself when complete (AutoClosesWhenComplete): the root is +// open and every step under it is closed. It answers nil when stepID belongs +// to no molecule, when the root is already closed or does not auto-close, or +// when a step is still open. It writes nothing; the caller closes the root, +// guarded on the RowVersion of the root it answers. +// +// It is AutoClosingMoleculeRoots then MoleculeStepsClosed, so a step under an +// ordinary epic, or under a root already closed, never reads the steps. +func CompletedMolecule(ctx context.Context, r MoleculeReader, stepID string) (*Issue, error) { + roots, err := AutoClosingMoleculeRoots(ctx, r, []string{stepID}) + if err != nil { + return nil, err + } + root := roots[stepID] + if root == nil { + return nil, nil + } + complete, err := MoleculeStepsClosed(ctx, r, root.ID) + if err != nil || !complete { + return nil, err + } + return root, nil +} + +// AutoClosingMoleculeRoots answers, for each id whose molecule root is open +// and closes itself when complete (AutoClosesWhenComplete), that root's row: +// the roots a close of ids could complete. It is the parent walk and the root +// rows MoleculeRoots reads, and nothing more — it never reads a molecule's +// steps — so a caller deciding several closes reads each root once and walks +// only the molecules that could auto-close. +func AutoClosingMoleculeRoots(ctx context.Context, r MoleculeReader, ids []string) (map[string]*Issue, error) { + rows, err := moleculeRootRows(ctx, r, ids) + if err != nil { + return nil, err + } + for id, root := range rows { + if root.Status == types.StatusClosed || !AutoClosesWhenComplete(root) { + delete(rows, id) + } + } + return rows, nil +} + +// MoleculeStepsClosed reports whether every parent-child descendant of rootID +// is closed — the steps LoadMolecule would load. It reads one level of steps +// at a time and answers false at the first step that is not closed, so an +// incomplete molecule costs the reads up to its first open step, not a load of +// the whole tree. A root with no steps answers true; a parent-child cycle +// terminates. +func MoleculeStepsClosed(ctx context.Context, r MoleculeReader, rootID string) (bool, error) { + if !r.usable() { + return false, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + seen := map[string]bool{rootID: true} + pending := []string{rootID} + for len(pending) > 0 { + parentID := pending[len(pending)-1] + pending = pending[:len(pending)-1] + children, err := r.Related(ctx, RelatedRequest{ + ID: parentID, + Direction: RelationIn, + Types: []types.DependencyType{types.DepParentChild}, + }) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return false, fmt.Errorf("reading the steps of %s: %w", parentID, err) + } + for _, child := range children { + if child == nil || child.DependencyType != types.DepParentChild || seen[child.ID] { + continue + } + if child.Status != types.StatusClosed { + return false, nil + } + seen[child.ID] = true + pending = append(pending, child.ID) + } + } + return true, nil +} + +// MoleculeAutoCloseReason is the close reason a molecule root's auto-close +// records. +const MoleculeAutoCloseReason = "all steps complete" + +// IsMoleculeAutoCloseRefusal reports whether err is a refusal of a completed +// molecule's root close — an outcome that leaves the root open without failing +// the step close that completed it: the close policy (a live blocker, +// ErrCloseBlocked; open children, *CloseOpenChildrenError) or a close guard (a +// template root, *TemplateReadOnlyError; a pinned root, *PinnedError; a root +// assigned to someone else, *CloseNotAssigneeError). The root close is +// unforced (MoleculeRootClose), so every guard holds for it exactly as it does +// for a direct close of the root. A guard only reads, before the close writes +// anything, and a policy refusal rolls back to the close's savepoint, so a +// refusal leaves the step close's transaction as it found it. +// +// ErrVersionMismatch on the root is NOT a refusal: should a backend's check +// ever see a revision its read did not, it fails the step's close +// (CloseRequest.AutoCloseMolecule). A write to the root that races the close +// does not produce it (MoleculeRootClose). +func IsMoleculeAutoCloseRefusal(err error) bool { + var ( + openChildren *CloseOpenChildrenError + template *TemplateReadOnlyError + pinned *PinnedError + notAssignee *CloseNotAssigneeError + ) + return errors.Is(err, ErrCloseBlocked) || + errors.As(err, &openChildren) || + errors.As(err, &template) || + errors.As(err, &pinned) || + errors.As(err, ¬Assignee) +} + +// MoleculeRootClose is the ONE close request a molecule auto-close sends for a +// root it found complete (AutoClosingMoleculeRoots, then MoleculeStepsClosed, +// read in the transaction that closes it): unforced, recording +// MoleculeAutoCloseReason and the step close's session, and guarded on the +// RowVersion of root as that read answered it. Every backend's +// in-transaction auto-close (CloseRequest.AutoCloseMolecule) builds its root +// close from it. +// +// The guard covers the root row's lifecycle state and nothing else (the +// version family's rule: it guards a row's lifecycle state, not its graph). +// Every backend reads the root and checks its revision in one transaction +// snapshot, so a status, assignee or started_at write that another +// transaction commits to the root after the read does not trip the guard. It +// conflicts with the root close at commit instead, and the backend's retry +// re-reads the root and decides again (a root another actor now holds is a +// refusal, and the step's close stands); only a close still conflicting when +// its retry budget runs out fails, with the serialization error. The guard +// does not cover the steps: reopening a step writes no root row, so a reopen +// that races the auto-close is not detected, and the root can close over the +// reopened step. +func MoleculeRootClose(root *Issue, actor, session string) CloseRequest { + version := root.RowVersion + return CloseRequest{ + Actor: actor, + IssueID: root.ID, + Reason: MoleculeAutoCloseReason, + Session: session, + ExpectedVersion: &version, + } +} diff --git a/issueops/molecule_active.go b/issueops/molecule_active.go new file mode 100644 index 000000000..f8a7b8c98 --- /dev/null +++ b/issueops/molecule_active.go @@ -0,0 +1,141 @@ +package issueops + +import ( + "context" + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/types" +) + +// InProgressMoleculeIDs answers the molecules with a step in progress for +// agent (every agent when agent is empty), each once, in the order the +// listing first reached a step of it. +// +// The listing is Reader.List over every plane and class a step can live in — +// templates, gates, infra and ephemeral rows, the four knobs rather than +// IncludeAllTypes, because the v0 wire publishes each knob and refuses the +// union. A read the backend refuses is returned, never answered as "none". +func InProgressMoleculeIDs(ctx context.Context, reader Reader, r MoleculeReader, agent string) ([]string, error) { + steps, err := listMoleculeCandidates(ctx, reader, types.StatusInProgress, agent) + if err != nil { + return nil, err + } + if len(steps) == 0 { + return nil, nil + } + ids := make([]string, len(steps)) + for i, step := range steps { + ids[i] = step.ID + } + roots, err := MoleculeRoots(ctx, r, ids) + if err != nil { + return nil, err + } + seen := map[string]bool{} + var out []string + for _, id := range ids { + if root := roots[id]; root != "" && !seen[root] { + seen[root] = true + out = append(out, root) + } + } + return out, nil +} + +// ActiveMoleculeIDs answers the molecules agent is working on, sorted: the +// ones with a step in progress or, when there are none, the ones its HOOKED +// work is attached to — a hooked epic itself, or an epic or template the +// hooked issue is blocked by. +func ActiveMoleculeIDs(ctx context.Context, reader Reader, r MoleculeReader, agent string) ([]string, error) { + ids, err := InProgressMoleculeIDs(ctx, reader, r, agent) + if err != nil { + return nil, fmt.Errorf("finding molecules in progress: %w", err) + } + if len(ids) == 0 { + ids, err = hookedMoleculeIDs(ctx, reader, r, agent) + if err != nil { + return nil, fmt.Errorf("finding hooked molecules: %w", err) + } + } + sort.Strings(ids) + return ids, nil +} + +func hookedMoleculeIDs(ctx context.Context, reader Reader, r MoleculeReader, agent string) ([]string, error) { + hooked, err := listMoleculeCandidates(ctx, reader, types.StatusHooked, agent) + if err != nil || len(hooked) == 0 { + return nil, err + } + seen := map[string]bool{} + var out []string + add := func(id string) { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + var attached []string + for _, issue := range hooked { + // A hooked epic IS the molecule (a patrol wisp hooked without a + // separate handoff bead). + if issue.IssueType == types.TypeEpic { + add(issue.ID) + continue + } + attached = append(attached, issue.ID) + } + if len(attached) == 0 { + return out, nil + } + edges, err := r.ReadEdges(ctx, EdgeReadRequest{IDs: attached, Types: []types.DependencyType{types.DepBlocks}}) + if err != nil { + return nil, fmt.Errorf("reading the blockers of hooked work: %w", err) + } + var blockers []string + for _, anchor := range edges.Anchors { + for _, edge := range anchor.Edges { + if edge != nil && edge.Type == types.DepBlocks { + blockers = append(blockers, edge.DependsOnID) + } + } + } + candidates, err := getIssues(ctx, r, blockers) + if err != nil { + return nil, err + } + for _, id := range blockers { + // An external or dangling blocker names nothing to load. + if candidate := candidates[id]; candidate != nil && + (candidate.IssueType == types.TypeEpic || hasLabel(candidate, MoleculeTemplateLabel)) { + add(id) + } + } + return out, nil +} + +func listMoleculeCandidates(ctx context.Context, reader Reader, status types.Status, agent string) ([]*Issue, error) { + if reader == nil { + return nil, fmt.Errorf("%w: listing molecule work needs a reader", ErrValidation) + } + unlimited := 0 + page, err := reader.List(ctx, ListRequest{ + Status: string(status), + Assignee: agent, + IncludeTemplates: true, + IncludeGates: true, + IncludeInfra: true, + IncludeEphemeral: true, + Limit: &unlimited, + }) + if err != nil { + return nil, err + } + out := make([]*Issue, 0, len(page.Items)) + for _, row := range page.Items { + if row != nil && row.Issue != nil { + out = append(out, row.Issue) + } + } + return out, nil +} diff --git a/issueops/molecule_internal_test.go b/issueops/molecule_internal_test.go new file mode 100644 index 000000000..0ffda4f16 --- /dev/null +++ b/issueops/molecule_internal_test.go @@ -0,0 +1,40 @@ +package issueops + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestBlockingDepths pins the depth rule the parallel grouping keys on: 0 with +// no open blocker, one more than the deepest open blocker otherwise. +func TestBlockingDepths(t *testing.T) { + root := &types.Issue{ID: "root", Status: types.StatusOpen} + step1 := &types.Issue{ID: "step1", Status: types.StatusOpen} + step2 := &types.Issue{ID: "step2", Status: types.StatusOpen} + step3 := &types.Issue{ID: "step3", Status: types.StatusOpen} + g := &MoleculeGraph{ + Root: root, + Issues: []*types.Issue{root, step1, step2, step3}, + IssueMap: map[string]*types.Issue{"root": root, "step1": step1, "step2": step2, "step3": step3}, + } + blockedBy := map[string]map[string]bool{ + "root": {}, + "step1": {"root": true}, + "step2": {"step1": true}, + "step3": {"step2": true}, + } + depths := blockingDepths(g, blockedBy) + for id, want := range map[string]int{"root": 0, "step1": 1, "step2": 2, "step3": 3} { + if depths[id] != want { + t.Errorf("%s depth = %d, want %d", id, depths[id], want) + } + } + + // A closed blocker no longer deepens what it blocked. + step1.Status = types.StatusClosed + depths = blockingDepths(g, blockedBy) + if depths["step2"] != 0 { + t.Errorf("step2 depth with its blocker closed = %d, want 0", depths["step2"]) + } +} diff --git a/issueops/molecule_test.go b/issueops/molecule_test.go new file mode 100644 index 000000000..296f168cc --- /dev/null +++ b/issueops/molecule_test.go @@ -0,0 +1,595 @@ +package issueops_test + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// fakeMolecules is an in-memory MoleculeReader, Reader (List), Claimer and +// Lifecycle (Update with Claim, Close) — just enough store for the molecule +// rules, so each rule is pinned without a database. +type fakeMolecules struct { + issueops.Reader + issueops.Lifecycle + issues map[string]*types.Issue + deps []*types.Dependency + + edgeErr error + listErr error + relatedErr error + // related records every Related read, by id, in order. + related []string + // claimErrs answers a claim of the keyed id with that error. + claimErrs map[string]error + claimed []string + updated []string + closes []issueops.CloseRequest + closeErr error +} + +func newFakeMolecules() *fakeMolecules { + return &fakeMolecules{issues: map[string]*types.Issue{}, claimErrs: map[string]error{}} +} + +func (f *fakeMolecules) add(id string, typ types.IssueType, status types.Status, labels ...string) *types.Issue { + issue := &types.Issue{ID: id, Title: "title " + id, IssueType: typ, Status: status, Labels: labels, RowVersion: int64(len(id) * 7)} + f.issues[id] = issue + return issue +} + +func (f *fakeMolecules) edge(from, to string, typ types.DependencyType) { + f.deps = append(f.deps, &types.Dependency{IssueID: from, DependsOnID: to, Type: typ}) +} + +func (f *fakeMolecules) GetMany(_ context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + for _, id := range req.IDs { + if issue := f.issues[id]; issue != nil { + clone := *issue + out.Issues = append(out.Issues, &clone) + } else { + out.Missing = append(out.Missing, id) + } + } + return out, nil +} + +func (f *fakeMolecules) Related(_ context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + f.related = append(f.related, req.ID) + if f.relatedErr != nil { + return nil, f.relatedErr + } + if f.issues[req.ID] == nil { + return nil, fmt.Errorf("%w: %s", issueops.ErrNotFound, req.ID) + } + var out []*issueops.RelatedIssue + for _, dep := range f.deps { + if dep.DependsOnID != req.ID || (len(req.Types) > 0 && !slices.Contains(req.Types, dep.Type)) { + continue + } + if issue := f.issues[dep.IssueID]; issue != nil { + out = append(out, &issueops.RelatedIssue{Issue: *issue, DependencyType: dep.Type}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (f *fakeMolecules) ReadEdges(_ context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + if f.edgeErr != nil { + return issueops.EdgeReadResult{}, f.edgeErr + } + var out issueops.EdgeReadResult + for _, id := range req.IDs { + anchor := issueops.AnchorEdges{ID: id, Missing: f.issues[id] == nil} + for _, dep := range f.deps { + if dep.IssueID == id && (len(req.Types) == 0 || slices.Contains(req.Types, dep.Type)) { + anchor.Edges = append(anchor.Edges, dep) + } + } + out.Anchors = append(out.Anchors, anchor) + } + return out, nil +} + +func (f *fakeMolecules) List(_ context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + if f.listErr != nil { + return issueops.IssuePage{}, f.listErr + } + var page issueops.IssuePage + for _, issue := range f.issues { + if string(issue.Status) == req.Status && (req.Assignee == "" || issue.Assignee == req.Assignee) { + page.Items = append(page.Items, &types.IssueWithCounts{Issue: issue}) + } + } + sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].ID < page.Items[j].ID }) + return page, nil +} + +func (f *fakeMolecules) take(id, actor string) (*types.Issue, error) { + if err := f.claimErrs[id]; err != nil { + return nil, err + } + issue := f.issues[id] + issue.Status, issue.Assignee = types.StatusInProgress, actor + clone := *issue + clone.Labels = nil + return &clone, nil +} + +func (f *fakeMolecules) Claim(_ context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + issue, err := f.take(req.IssueID, req.Actor) + if err != nil { + return issueops.ClaimResult{}, err + } + f.claimed = append(f.claimed, req.IssueID) + return issueops.ClaimResult{Issue: issue, Changed: true}, nil +} + +func (f *fakeMolecules) Update(_ context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + if !req.Claim { + return issueops.UpdateResult{}, errors.New("fake: only claims") + } + issue, err := f.take(req.IssueID, req.Actor) + if err != nil { + return issueops.UpdateResult{}, err + } + f.updated = append(f.updated, req.IssueID) + return issueops.UpdateResult{Issue: issue, Changed: true}, nil +} + +func (f *fakeMolecules) Close(_ context.Context, req issueops.CloseRequest) (issueops.CloseResult, error) { + f.closes = append(f.closes, req) + if f.closeErr != nil { + return issueops.CloseResult{}, f.closeErr + } + issue := f.issues[req.IssueID] + if req.ExpectedVersion != nil && *req.ExpectedVersion != issue.RowVersion { + return issueops.CloseResult{}, issueops.ErrVersionMismatch + } + issue.Status = types.StatusClosed + clone := *issue + return issueops.CloseResult{Issue: &clone, Changed: true}, nil +} + +func (f *fakeMolecules) reader() issueops.MoleculeReader { return issueops.MoleculeReaderOver(f) } + +func (f *fakeMolecules) stepper() issueops.MoleculeStepper { + return issueops.NewMoleculeStepper(issueops.MoleculeStepperRoles{Reader: f.reader(), Claimer: f, Lifecycle: f}) +} + +// A molecule: root (molecule type) <- a, b, c; c blocked by b; b blocked by a. +func seedChain(f *fakeMolecules) { + f.add("m", types.TypeMolecule, types.StatusOpen) + f.add("m.a", types.TypeTask, types.StatusClosed) + f.add("m.b", types.TypeTask, types.StatusOpen) + f.add("m.c", types.TypeTask, types.StatusOpen) + for _, id := range []string{"m.a", "m.b", "m.c"} { + f.edge(id, "m", types.DepParentChild) + } + f.edge("m.b", "m.a", types.DepBlocks) + f.edge("m.c", "m.b", types.DepBlocks) +} + +func TestMoleculeRootsWalksToTheTopmostMoleculeRoot(t *testing.T) { + f := newFakeMolecules() + f.add("root", types.TypeEpic, types.StatusOpen) + f.add("child", types.TypeTask, types.StatusOpen) + f.add("grandchild", types.TypeTask, types.StatusOpen) + f.add("orphan", types.TypeTask, types.StatusOpen) + f.add("plain", types.TypeTask, types.StatusOpen) + f.add("plainkid", types.TypeTask, types.StatusOpen) + f.add("tmpl", types.TypeTask, types.StatusOpen, issueops.MoleculeTemplateLabel) + f.add("tmplkid", types.TypeTask, types.StatusOpen) + f.edge("child", "root", types.DepParentChild) + f.edge("grandchild", "child", types.DepParentChild) + f.edge("plainkid", "plain", types.DepParentChild) + f.edge("tmplkid", "tmpl", types.DepParentChild) + + roots, err := issueops.MoleculeRoots(context.Background(), f.reader(), []string{"grandchild", "child", "root", "orphan", "plainkid", "tmplkid", "absent"}) + if err != nil { + t.Fatalf("MoleculeRoots: %v", err) + } + want := map[string]string{"grandchild": "root", "child": "root", "root": "root", "tmplkid": "tmpl"} + if len(roots) != len(want) { + t.Fatalf("MoleculeRoots = %v, want %v", roots, want) + } + for id, root := range want { + if roots[id] != root { + t.Errorf("MoleculeRoots[%s] = %q, want %q", id, roots[id], root) + } + } +} + +// Every read the rules make RETURNS a refusal. Answering "not in a molecule" +// or "no molecules" for a read that never happened is how `bd mol current` +// went silently empty and a molecule root was stranded open. +func TestMoleculeReadsReturnRefusals(t *testing.T) { + ctx := context.Background() + refusal := errors.New("refused") + + f := newFakeMolecules() + seedChain(f) + f.edgeErr = refusal + if _, err := issueops.MoleculeRoots(ctx, f.reader(), []string{"m.b"}); !errors.Is(err, refusal) { + t.Errorf("MoleculeRoots with a refused edge read = %v, want the refusal", err) + } + if _, err := issueops.CompletedMolecule(ctx, f.reader(), "m.b"); !errors.Is(err, refusal) { + t.Errorf("CompletedMolecule with a refused edge read = %v, want the refusal", err) + } + + f = newFakeMolecules() + seedChain(f) + f.relatedErr = refusal + if _, err := issueops.ViewMolecule(ctx, f.reader(), "m"); !errors.Is(err, refusal) { + t.Errorf("ViewMolecule with a refused relations read = %v, want the refusal", err) + } + if _, err := issueops.ReadMoleculeProgress(ctx, f.reader(), "m"); !errors.Is(err, refusal) { + t.Errorf("ReadMoleculeProgress with a refused relations read = %v, want the refusal", err) + } + + f = newFakeMolecules() + f.listErr = refusal + if _, err := issueops.ActiveMoleculeIDs(ctx, f, f.reader(), "alice"); !errors.Is(err, refusal) { + t.Errorf("ActiveMoleculeIDs with a refused listing = %v, want the refusal", err) + } + if _, err := issueops.InProgressMoleculeIDs(ctx, f, f.reader(), "alice"); !errors.Is(err, refusal) { + t.Errorf("InProgressMoleculeIDs with a refused listing = %v, want the refusal", err) + } +} + +func TestMoleculeViewDerivesStatesProgressAndTheNextStep(t *testing.T) { + f := newFakeMolecules() + seedChain(f) + f.add("m.d", types.TypeTask, types.StatusInProgress) + f.edge("m.d", "m", types.DepParentChild) + f.issues["m.d"].Assignee = "alice" + + view, err := issueops.ViewMolecule(context.Background(), f.reader(), "m") + if err != nil { + t.Fatalf("ViewMolecule: %v", err) + } + if view.Total != 4 || view.Completed != 1 || view.Complete() { + t.Errorf("progress = %d/%d complete=%v, want 1/4 incomplete", view.Completed, view.Total, view.Complete()) + } + states := map[string]string{} + for _, step := range view.Steps { + states[step.Issue.ID] = step.State + } + want := map[string]string{"m.a": "done", "m.b": "ready", "m.c": "pending", "m.d": "current"} + for id, state := range want { + if states[id] != state { + t.Errorf("state[%s] = %q, want %q", id, states[id], state) + } + } + if view.CurrentStep == nil || view.CurrentStep.ID != "m.d" || view.CurrentStep.Assignee != "alice" { + t.Errorf("CurrentStep = %+v, want m.d assigned to alice (the full step row)", view.CurrentStep) + } + if view.NextStep == nil || view.NextStep.ID != "m.b" { + t.Errorf("NextStep = %+v, want m.b", view.NextStep) + } + // Ready is ready --mol's list: every analysis-ready node, hydrated. + ready := map[string]*issueops.Issue{} + for _, issue := range view.Ready { + ready[issue.ID] = issue + } + if ready["m.b"] == nil || ready["m.d"] == nil || ready["m.a"] != nil || ready["m.c"] != nil { + t.Errorf("Ready = %v, want m.b and m.d, never the closed m.a or the blocked m.c", view.Ready) + } + if d := ready["m.d"]; d != nil && d.Assignee != "alice" { + t.Errorf("Ready m.d assignee = %q, want alice (the full step row)", d.Assignee) + } + // Ordered by in-molecule blockers, fewest first: m.c waits on one step. + if last := view.Steps[len(view.Steps)-1].Issue.ID; last != "m.c" && last != "m.b" { + t.Errorf("last step = %s, want a step that waits on a blocker", last) + } + progress, err := issueops.ReadMoleculeProgress(context.Background(), f.reader(), "m") + if err != nil { + t.Fatalf("ReadMoleculeProgress: %v", err) + } + if progress.Total != 4 || progress.Completed != 1 || progress.InProgress != 1 || progress.CurrentStepID != "m.d" { + t.Errorf("progress = %+v, want 4 total, 1 completed, 1 in progress (m.d)", progress) + } +} + +func TestCompletedMoleculeAnswersOnlyAnOpenCompleteAutoClosingRoot(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + rootType types.IssueType + labels []string + closeAll bool + want bool + }{ + {"molecule root, all steps closed", types.TypeMolecule, nil, true, true}, + {"template epic, all steps closed", types.TypeEpic, []string{issueops.MoleculeTemplateLabel}, true, true}, + {"plain epic stays open for explicit close", types.TypeEpic, nil, true, false}, + {"a step still open", types.TypeMolecule, nil, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMolecules() + f.add("r", tc.rootType, types.StatusOpen, tc.labels...) + f.add("r.1", types.TypeTask, types.StatusClosed) + status := types.StatusOpen + if tc.closeAll { + status = types.StatusClosed + } + f.add("r.2", types.TypeTask, status) + f.edge("r.1", "r", types.DepParentChild) + f.edge("r.2", "r", types.DepParentChild) + root, err := issueops.CompletedMolecule(ctx, f.reader(), "r.1") + if err != nil { + t.Fatalf("CompletedMolecule: %v", err) + } + if (root != nil) != tc.want { + t.Fatalf("CompletedMolecule = %v, want completed=%v", root, tc.want) + } + }) + } + + f := newFakeMolecules() + f.add("orphan", types.TypeTask, types.StatusClosed) + if root, err := issueops.CompletedMolecule(ctx, f.reader(), "orphan"); err != nil || root != nil { + t.Errorf("CompletedMolecule(orphan) = %v, %v, want nil, nil", root, err) + } +} + +// A close under a root that cannot auto-close — an ordinary epic, a root +// already closed — reads no steps at all, and an eligible root's steps are read +// only up to the first open one. +func TestCompletedMoleculeReadsOnlyTheStepsItMust(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + rootType types.IssueType + status types.Status + }{ + {"ordinary epic", types.TypeEpic, types.StatusOpen}, + {"closed molecule root", types.TypeMolecule, types.StatusClosed}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMolecules() + f.add("r", tc.rootType, tc.status) + for _, id := range []string{"r.1", "r.2"} { + f.add(id, types.TypeTask, types.StatusClosed) + f.edge(id, "r", types.DepParentChild) + } + root, err := issueops.CompletedMolecule(ctx, f.reader(), "r.1") + if err != nil || root != nil { + t.Fatalf("CompletedMolecule = %v, %v, want nil, nil", root, err) + } + if len(f.related) != 0 { + t.Errorf("steps read %v, want none: this root never auto-closes", f.related) + } + }) + } + + // r.1 is open; r.2 and r.3 are closed and have steps of their own. The + // walk answers at r.1 without reading below r.2 or r.3. + f := newFakeMolecules() + f.add("r", types.TypeMolecule, types.StatusOpen) + f.add("r.1", types.TypeTask, types.StatusOpen) + f.edge("r.1", "r", types.DepParentChild) + for _, id := range []string{"r.2", "r.3"} { + f.add(id, types.TypeTask, types.StatusClosed) + f.edge(id, "r", types.DepParentChild) + f.add(id+".1", types.TypeTask, types.StatusClosed) + f.edge(id+".1", id, types.DepParentChild) + } + root, err := issueops.CompletedMolecule(ctx, f.reader(), "r.2") + if err != nil || root != nil { + t.Fatalf("CompletedMolecule = %v, %v, want nil, nil (r.1 is open)", root, err) + } + if !slices.Equal(f.related, []string{"r"}) { + t.Errorf("steps read %v, want only r's: r.1 is open, so nothing below r is read", f.related) + } +} + +// MoleculeStepsClosed covers every descendant, not just the direct steps, and a +// parent-child cycle terminates. +func TestMoleculeStepsClosedCoversEveryDescendant(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + f.add("r", types.TypeMolecule, types.StatusOpen) + f.add("r.1", types.TypeTask, types.StatusClosed) + f.edge("r.1", "r", types.DepParentChild) + f.add("r.1.1", types.TypeTask, types.StatusOpen) + f.edge("r.1.1", "r.1", types.DepParentChild) + if closed, err := issueops.MoleculeStepsClosed(ctx, f.reader(), "r"); err != nil || closed { + t.Errorf("MoleculeStepsClosed with r.1.1 open = %v, %v, want false", closed, err) + } + + f.issues["r.1.1"].Status = types.StatusClosed + f.edge("r", "r.1.1", types.DepParentChild) // a cycle back to the root + f.related = nil + if closed, err := issueops.MoleculeStepsClosed(ctx, f.reader(), "r"); err != nil || !closed { + t.Errorf("MoleculeStepsClosed with every step closed = %v, %v, want true", closed, err) + } + if len(f.related) != 3 { + t.Errorf("steps read %v, want r, r.1 and r.1.1 once each", f.related) + } + + f.relatedErr = errors.New("refused") + if _, err := issueops.MoleculeStepsClosed(ctx, f.reader(), "r"); !errors.Is(err, f.relatedErr) { + t.Errorf("MoleculeStepsClosed with a refused read = %v, want the refusal", err) + } +} + +// The root close's refusals that leave the step's close standing are the close +// policy and the close guards; a lost race on the root's revision is not one. +func TestIsMoleculeAutoCloseRefusalIsPolicyAndGuardsNeverAVersionRace(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"live blocker", &issueops.BlockedError{IssueID: "r", Blockers: []issueops.Blocker{{ID: "b"}}, Err: issueops.ErrCloseBlocked}, true}, + {"open children", &issueops.CloseOpenChildrenError{IssueID: "r", OpenChildren: 1}, true}, + {"template root", &issueops.TemplateReadOnlyError{IssueID: "r"}, true}, + {"pinned root", &issueops.PinnedError{IssueID: "r"}, true}, + {"root assigned to someone else", &issueops.CloseNotAssigneeError{IssueID: "r", Assignee: "bob", Actor: "alice"}, true}, + {"wrapped guard", fmt.Errorf("close r: %w", &issueops.CloseNotAssigneeError{IssueID: "r", Assignee: "bob", Actor: "alice"}), true}, + {"lost race on the root revision", issueops.ErrVersionMismatch, false}, + {"typed lost race", &issueops.VersionMismatchError{Expected: 1, Current: 2}, false}, + {"wrapped lost race", fmt.Errorf("close r: %w", &issueops.VersionMismatchError{Expected: 1, Current: 2}), false}, + {"storage failure", errors.New("connection reset"), false}, + {"no error", nil, false}, + } { + if got := issueops.IsMoleculeAutoCloseRefusal(tc.err); got != tc.want { + t.Errorf("%s: IsMoleculeAutoCloseRefusal(%v) = %v, want %v", tc.name, tc.err, got, tc.want) + } + } +} + +func TestMoleculeRootCloseIsUnforcedAndGuardedOnTheRootRevision(t *testing.T) { + root := &issueops.Issue{ID: "r", RowVersion: 41} + got := issueops.MoleculeRootClose(root, "alice", "sess") + if got.ExpectedVersion == nil || *got.ExpectedVersion != 41 || got.Force { + t.Errorf("root close = %+v, want unforced and guarded on the root revision 41", got) + } + if got.IssueID != "r" || got.Reason != issueops.MoleculeAutoCloseReason || got.Session != "sess" || got.Actor != "alice" { + t.Errorf("root close = %+v, want root r, the auto-close reason, the session and the actor", got) + } + root.RowVersion = 42 + if *got.ExpectedVersion != 41 { + t.Error("the guard aliases the root's revision; it must be the revision read") + } +} + +func TestMoleculeStepperClaimsTheNextReadyStepForTheActor(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + seedChain(f) + + // Without AutoClaim nothing is written. + res, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.a"}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if res.MoleculeID != "m" || res.Claimed || res.NextStep == nil || res.NextStep.ID != "m.b" || len(f.claimed) != 0 { + t.Fatalf("Advance without a claim = %+v (claimed %v), want next m.b and no claim", res, f.claimed) + } + + res, err = f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.a", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || res.NextStep == nil || res.NextStep.ID != "m.b" || res.NextStep.Assignee != "alice" || + res.NextStep.Status != types.StatusInProgress { + t.Fatalf("Advance = %+v, want m.b claimed for alice", res) + } + if !slices.Equal(f.claimed, []string{"m.b"}) || len(f.updated) != 0 { + t.Errorf("claims = %v / updates = %v, want one Claimer claim of m.b", f.claimed, f.updated) + } +} + +// A ready step another actor holds is skipped, never taken; any other claim +// failure is returned rather than read as a lost race. +func TestMoleculeStepperSkipsAHeldStepAndReturnsOtherFailures(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + f.add("m", types.TypeMolecule, types.StatusOpen) + for _, id := range []string{"m.1", "m.2"} { + f.add(id, types.TypeTask, types.StatusOpen) + f.edge(id, "m", types.DepParentChild) + } + f.add("m.0", types.TypeTask, types.StatusClosed) + f.edge("m.0", "m", types.DepParentChild) + f.claimErrs["m.1"] = &issueops.ClaimConflictError{Err: issueops.ErrAlreadyClaimed} + + res, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.0", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || res.NextStep.ID != "m.2" { + t.Fatalf("Advance = %+v, want m.1 skipped (held) and m.2 claimed", res) + } + + f.issues["m.2"].Status, f.issues["m.2"].Assignee = types.StatusOpen, "" + boom := errors.New("connection reset") + f.claimErrs["m.1"] = boom + if _, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.0", AutoClaim: true}); !errors.Is(err, boom) { + t.Fatalf("Advance with a failed claim = %v, want the failure", err) + } + + f.claimErrs["m.1"] = issueops.ErrNotClaimable + f.claimErrs["m.2"] = issueops.ErrAlreadyClaimed + res, err = f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.0", AutoClaim: true}) + if err != nil || res.Claimed || res.NextStep == nil || res.NextStep.ID != "m.1" { + t.Fatalf("Advance with every ready step lost = %+v, %v; want no claim and the first ready step named", res, err) + } +} + +func TestMoleculeStepperClaimsAWispStepThroughLifecycle(t *testing.T) { + f := newFakeMolecules() + f.add("w", types.TypeMolecule, types.StatusOpen).Ephemeral = true + f.add("w.1", types.TypeTask, types.StatusClosed).Ephemeral = true + f.add("w.2", types.TypeTask, types.StatusOpen).Ephemeral = true + f.edge("w.1", "w", types.DepParentChild) + f.edge("w.2", "w", types.DepParentChild) + res, err := f.stepper().Advance(context.Background(), issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "w.1", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || !slices.Equal(f.updated, []string{"w.2"}) || len(f.claimed) != 0 { + t.Fatalf("Advance over a wisp = %+v (updates %v, claims %v), want w.2 claimed through Lifecycle", res, f.updated, f.claimed) + } +} + +func TestMoleculeStepperOutsideAMoleculeAndWhenComplete(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + f.add("orphan", types.TypeTask, types.StatusClosed) + res, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "orphan", AutoClaim: true}) + if err != nil || res.MoleculeID != "" || res.ClosedStep == nil || res.NextStep != nil { + t.Fatalf("Advance outside a molecule = %+v, %v; want no molecule", res, err) + } + + f.add("m", types.TypeMolecule, types.StatusOpen) + f.add("m.1", types.TypeTask, types.StatusClosed) + f.edge("m.1", "m", types.DepParentChild) + res, err = f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.1", AutoClaim: true}) + if err != nil || !res.Complete || res.NextStep != nil { + t.Fatalf("Advance on a complete molecule = %+v, %v; want complete", res, err) + } + + if _, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "absent"}); !errors.Is(err, issueops.ErrNotFound) { + t.Errorf("Advance on an absent step = %v, want ErrNotFound", err) + } + for _, req := range []issueops.AdvanceRequest{{ClosedStepID: "m.1"}, {Actor: "alice"}} { + if _, err := f.stepper().Advance(ctx, req); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Advance(%+v) = %v, want ErrValidation", req, err) + } + } +} + +func TestActiveMoleculeIDsFallsBackToHookedWork(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + seedChain(f) + f.issues["m.b"].Status, f.issues["m.b"].Assignee = types.StatusInProgress, "alice" + ids, err := issueops.ActiveMoleculeIDs(ctx, f, f.reader(), "alice") + if err != nil || !slices.Equal(ids, []string{"m"}) { + t.Fatalf("ActiveMoleculeIDs = %v, %v, want [m]", ids, err) + } + + g := newFakeMolecules() + g.add("epic", types.TypeEpic, types.StatusOpen) + g.add("handoff", types.TypeTask, types.StatusHooked).Assignee = "bob" + g.edge("handoff", "epic", types.DepBlocks) + ids, err = issueops.ActiveMoleculeIDs(ctx, g, g.reader(), "bob") + if err != nil || !slices.Equal(ids, []string{"epic"}) { + t.Fatalf("ActiveMoleculeIDs (hooked) = %v, %v, want [epic]", ids, err) + } + ids, err = issueops.ActiveMoleculeIDs(ctx, g, g.reader(), "carol") + if err != nil || len(ids) != 0 { + t.Fatalf("ActiveMoleculeIDs (nobody's work) = %v, %v, want none", ids, err) + } +} diff --git a/issueops/moleculestepper.go b/issueops/moleculestepper.go new file mode 100644 index 000000000..d749cae88 --- /dev/null +++ b/issueops/moleculestepper.go @@ -0,0 +1,209 @@ +package issueops + +import ( + "context" + "errors" + "fmt" +) + +// AdvanceRequest describes moving a molecule on after one of its steps +// closed: the operation behind `bd close --continue`. +type AdvanceRequest struct { + // Actor is who advances. With AutoClaim it becomes the claimed step's + // assignee, exactly as ClaimRequest.Actor does. It must not be empty. + Actor string + // ClosedStepID names the step that closed. It must not be empty, and an + // id that names nothing is ErrNotFound. Nothing requires the step to be + // closed: advancing reads where the molecule stands now. + ClosedStepID string + // AutoClaim claims the next ready step for Actor. Without it the advance + // writes nothing and only names the step. + AutoClaim bool +} + +// AdvanceResult reports where a molecule stands after the advance. +type AdvanceResult struct { + // ClosedStep is the step the request named, as read before any claim. + ClosedStep *Issue + // MoleculeID is the step's molecule root, or "" when the step belongs to + // no molecule — in which case nothing else below is set. + MoleculeID string + // Complete reports that every step of the molecule is closed. NextStep is + // then nil. + Complete bool + // NextStep is the step the advance claimed or, without a claim, the first + // ready step (see MoleculeView.ReadySteps). Nil when no step is ready. + NextStep *Issue + // Claimed reports that the advance claimed NextStep for Actor: NextStep is + // then the post-claim row, in progress and assigned to Actor. + Claimed bool +} + +// MoleculeStepper describes advancing a molecule to its next ready step. Like +// Lifecycle and Claimer it is a role with its own accessor; a new capability +// gets a new role interface, so never append a method here. +// +// THE CLAIM IS A REAL CLAIM. A durable step is claimed through the Claimer +// role and a wisp step through Lifecycle.Update's Claim (the Claimer does not +// serve the wisp plane; both run the one claim compare-and-set). Either way +// the step's assignee becomes Actor and a step another actor holds is refused +// rather than taken. A ready step lost that way — held by someone else, or +// moved out of a claimable status since it was read — is skipped for the next +// ready step; when every ready step is lost the advance claims nothing and +// still reports the first one as NextStep. Any other claim failure is +// returned. +// +// Implementations never mutate the caller's request. Validation failures +// match ErrValidation and write nothing. +type MoleculeStepper interface { + Advance(ctx context.Context, req AdvanceRequest) (AdvanceResult, error) +} + +// MoleculeStepperRoles is the role set the shared stepper is composed over. +type MoleculeStepperRoles struct { + Reader MoleculeReader + Claimer Claimer + Lifecycle Lifecycle +} + +// moleculeStepperSource is the accessor set a store or unit-of-work provider +// offers for the roles NewMoleculeStepper composes. +type moleculeStepperSource interface { + BatchGetter() (BatchGetter, error) + IssueRelations() (Relations, error) + EdgeReader() (EdgeReader, error) + IssueClaimer() (Claimer, error) + IssueLifecycle() (Lifecycle, error) +} + +// MoleculeStepperRolesOf binds the stepper's roles to src's own accessors. +func MoleculeStepperRolesOf(src moleculeStepperSource) (MoleculeStepperRoles, error) { + reader, err := NewMoleculeReader(src) + if err != nil { + return MoleculeStepperRoles{}, err + } + claimer, err := src.IssueClaimer() + if err != nil { + return MoleculeStepperRoles{}, err + } + lifecycle, err := src.IssueLifecycle() + if err != nil { + return MoleculeStepperRoles{}, err + } + return MoleculeStepperRoles{Reader: reader, Claimer: claimer, Lifecycle: lifecycle}, nil +} + +// NewMoleculeStepper is THE MoleculeStepper implementation. Every store +// backend's accessor answers it composed over that backend's own roles. The +// http client's accessor dials the server's issues.advanceMolecule instead, +// where the server's library runs this stepper, and refuses before dialing a +// server that does not advertise the operation; it never composes a copy over +// the client's roles. So every route advances a molecule by one rule. +func NewMoleculeStepper(roles MoleculeStepperRoles) MoleculeStepper { + return moleculeStepper{roles: roles} +} + +type moleculeStepper struct{ roles MoleculeStepperRoles } + +// ValidateAdvanceRequest applies the request rules every MoleculeStepper +// shares. +func ValidateAdvanceRequest(req AdvanceRequest) error { + if req.Actor == "" { + return fmt.Errorf("%w: advance requires an actor", ErrValidation) + } + if req.ClosedStepID == "" { + return fmt.Errorf("%w: advance requires the closed step id", ErrValidation) + } + return nil +} + +func (s moleculeStepper) Advance(ctx context.Context, req AdvanceRequest) (AdvanceResult, error) { + if err := ValidateAdvanceRequest(req); err != nil { + return AdvanceResult{}, err + } + r := s.roles.Reader + if !r.usable() || s.roles.Claimer == nil || s.roles.Lifecycle == nil { + return AdvanceResult{}, fmt.Errorf("%w: molecule stepper is missing a role", ErrValidation) + } + steps, err := getIssues(ctx, r, []string{req.ClosedStepID}) + if err != nil { + return AdvanceResult{}, err + } + closed := steps[req.ClosedStepID] + if closed == nil { + return AdvanceResult{}, fmt.Errorf("%w: issue %s", ErrNotFound, req.ClosedStepID) + } + result := AdvanceResult{ClosedStep: closed} + + rootID, err := MoleculeRoot(ctx, r, req.ClosedStepID) + if err != nil { + return AdvanceResult{}, fmt.Errorf("finding the molecule of %s: %w", req.ClosedStepID, err) + } + if rootID == "" { + return result, nil + } + result.MoleculeID = rootID + + view, err := ViewMolecule(ctx, r, rootID) + if err != nil { + return AdvanceResult{}, fmt.Errorf("loading molecule %s: %w", rootID, err) + } + if view.Complete() { + result.Complete = true + return result, nil + } + ready := view.ReadySteps() + if len(ready) == 0 { + return result, nil + } + result.NextStep = ready[0] + if !req.AutoClaim { + return result, nil + } + for _, candidate := range ready { + claimed, err := s.claimStep(ctx, candidate, req.Actor) + if isLostStepClaim(err) { + continue + } + if err != nil { + return AdvanceResult{}, fmt.Errorf("claiming step %s: %w", candidate.ID, err) + } + result.NextStep = claimed + result.Claimed = true + break + } + return result, nil +} + +// claimStep claims one step for actor and answers its post-claim row, labels +// carried over from the row the view read (the claim answers the bare row). +func (s moleculeStepper) claimStep(ctx context.Context, step *Issue, actor string) (*Issue, error) { + var post *Issue + if step.Ephemeral || step.NoHistory { + res, err := s.roles.Lifecycle.Update(ctx, UpdateRequest{Actor: actor, IssueID: step.ID, Claim: true}) + if err != nil { + return nil, err + } + post = res.Issue + } else { + res, err := s.roles.Claimer.Claim(ctx, ClaimRequest{Actor: actor, IssueID: step.ID}) + if err != nil { + return nil, err + } + post = res.Issue + } + if post == nil { + return step, nil + } + if len(post.Labels) == 0 && len(step.Labels) > 0 { + post.Labels = append([]string(nil), step.Labels...) + } + return post, nil +} + +// isLostStepClaim reports whether a step claim failed because the step is no +// longer this actor's to take: held by another actor, or moved out of a +// claimable status since the view read it. +func isLostStepClaim(err error) bool { + return err != nil && (errors.Is(err, ErrAlreadyClaimed) || errors.Is(err, ErrNotClaimable)) +} From ec55baa425fb0ddcf226bb84ac0114fe028b7189 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 13 Oct 2026 04:12:54 +0000 Subject: [PATCH 10/99] fix(http): bd show --current, recompute-blocked and search over http through issueops entries (S6e/S25/S26) (#7528) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(show): --current through issueops.FindCurrentIssue over the Querier role; no swallowed refusal (S6e) Over http `bd show --current` printed the WRONG issue, exit 0. resolveCurrentIssueID asked the raw SearchIssues method for the actor's in_progress and then hooked work; the http backend refuses that method, the refusal was taken for "nothing found", and the command fell back to the last-touched issue. The proxied route had its own copy of the lookup, read inside its unit of work, with the same swallow. One library entry now: issueops.FindCurrentIssue(ctx, Querier, actor). It asks the Querier role (queryIssues over http, which every deployed server publishes) for `status= AND assignee=""`, limit 1 — exactly the raw filter, answered in the store's default order, so the same row comes first on every backend — in_progress, then hooked. A failed read is an error; "" (two reads that answered and found nothing) is the only miss. The actor is a quoted, escaped query value; the query language reads `assignee=none`/`null` as "unassigned", so such an actor (or an empty one) is not asked about. Both CLI routes call it with their route's Querier (store.Querier() or the proxied provider's) through one cmd/bd helper; what stays in the CLI is the actor and the last-touched fallback (a workspace file). The proxied route resolves --current before it opens its unit of work, and its duplicate in-transaction lookup is deleted. Tests: issueops unit tests (order, failure is an error on either read, quoting, unaskable actors, each expression evaluates to exactly the raw filter); cmd/bd fallback tests; served e2e TestE2E_ShowCurrentOverHTTP (in-progress beats a later-touched issue, hooked when nothing is in progress, last-touched only when neither). Against the old code the e2e fails with exactly the reported bug. Co-Authored-By: Claude Opus 5.5 * fix(recompute-blocked): one library entry, storage.RecomputeBlocked; a no-op on a remote backend (S25) `bd recompute-blocked` refused over http ("storage backend does not support is_blocked recompute"), and gc's BdStore.RecomputeBlocked issues it. is_blocked is server-derived. Every write a remote client makes is committed by the server's own store, which keeps the flag under issueops.BlockedStateInvariant's local-write clause (targeted, inside the write's transaction; a Postgres-family server has no full recompute at all and runs a no-op readiness strategy for the same reason). A client cannot leave the flag stale and holds no copy of it. The invariant's merge clause (a recompute skipped after a pull, bd-6dnrw.37; a migration that over-set it, beads#7037) happens to the SERVER's database, and its repair is this command in the server's workspace. So the decision is in the library, not the CLI: storage.RecomputeBlocked (internal/storage/blocked_recomputer.go) is the one entry every backend goes through. A local store (storage.BlockedRecomputer) recomputes its column; a store implementing storage.ServerMaintainedBlockedState — the http client, whose BlockedStateMaintainedByServer answers true — recomputes nothing and reports MaintainedBy "server". No wire operation: a whole-column rewrite any bearer of a workspace credential could trigger, for state no client can have caused, is not one to publish. The proxied route keeps its pinned-connection plumbing around the same versioncontrolops body the stores run. --json keeps rows_corrected (gc requires the member; wh-bridge-sync reads the exit code) as 0 and adds maintained_by: "server"; the text says where the repair runs. Tests: storage unit tests (local repair runs, server-maintained wins even over a local repair and runs none, neither refuses); served e2e TestE2E_RecomputeBlockedOverHTTP (exit 0, JSON shape, text, ready set unchanged). With the server-maintained branch disabled the e2e fails with the old refusal. Co-Authored-By: Claude Opus 5.5 * feat(search): every bd search route through Reader.List with issueops.SearchListRequest; listIssues `q` behind issues.list.search (S26) `bd search` called the raw SearchIssues method with its text, which the http backend refuses ("partial-id search is not available over HTTP"), and its direct and proxied routes each mapped the flags onto a storage filter by hand. No served operation could carry search's meaning (title OR id contains the text, case-insensitively; an id-shaped text also matches exact id, id prefix and external ref): TitleSearch is title-only and the query language has no title-or-id predicate. The library entry is issueops.Reader.List: - issueops.ListRequest gains Query, SearchIssues' free-text argument as a member. Both Reader bodies (workapi/storereader and the uow reader) pass it to SearchIssuesWithCounts where they passed "". It narrows and lifts no default exclusion; the ready arm refuses it (readyScopeFields). - issueops.SearchListRequest(query) is search's scope as a request: Query, every status (AllFlag), every kind of row (the four Include*), and the store's default order (SortBy priority), so --limit keeps the rows a raw read kept. - CLI: ONE runSearch for the direct, proxied and remote routes. One builder (searchListRequest) maps the flags onto SearchListRequest; the route only picks the reader (store.IssueReader() or the proxied provider's). --sort/--reverse reorder the page as before. search_proxied_server.go and its hand-built filter are deleted, and search leaves the forbidigo filter-writing exemption. - Server: GET /v0/beads/issues decodes `q` into ListRequest.Query and calls the same Reader role, behind the new issues.list.search behavior token (spec parameter cites it; additive, no wire_revision bump; apigen and the wire-shape digest regenerated). - Client: the encoder maps Query to `q`; httpReader.List refuses a request carrying Query BEFORE dialing, with ErrUnsupported{Capability: issues.list.search}, unless the handshake advertises it — never a round trip an old server would 400, never the unsearched listing passed off as the search. CapListSearch mirrors the server token. Flags listIssues publishes nothing for are put on the request so the encoder refuses them rather than widening the search. Tests: Reader contract case RunReaderListQueryMatchesTheTitleOrTheID (also pins SearchListRequest's scope) on all four legs — dolt, embedded dolt, uow, served http (label-scoped, so the served leg runs it as written); handler test that `q` reaches the seam as the search text, not TitleSearch; client tests for `q` and the pre-dial refusal (zero dials); round-trip and vocabulary gates; TestSearchRoutesAgree runs the same twelve flag sets on the direct, proxied and remote routes and requires identical answers (remote is bd serve over the proxied database, so ids and order too); served e2e TestE2E_SearchOverHTTP. The remote legs skip with the SearchIssuesWithCounts refusal until S6c's externaldeps reader passthrough is underneath (loud refusal, never an empty or wrong answer); with S6c cherry-picked all three routes agree. Co-Authored-By: Claude Opus 5.5 * build: resync BUILD srcs and the embedded storage shard manifest for S6e/S25/S26 gazelle picks up cmd/bd/show_current.go, cmd/bd/search_remote.go, the three backend/http e2e files and internal/httpclient/list_search_test.go; the embedded storage shard manifest gains TestEmbeddedReaderListQueryMatchesTheTitleOrTheID (scripts/ci/gen_embedded_storage_shard_manifest.py 15 --weights=duration --write). Co-Authored-By: Claude Opus 5.5 * style: gofmt list_show_filter_modes.go (trailing blank line) Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): drop the show --current and recompute-blocked marks S6e and S25 retire With S26 folded in, `bd show --current` resolves through issueops.FindCurrentIssue over the Querier role and `bd recompute-blocked` goes through storage.RecomputeBlocked (a no-op on a remote backend), so gc-show-current and gc-recompute-blocked lose their pending marks. mc-search stays pending, relabelled S6c+SRV-1: bd search now asks listIssues with q behind issues.list.search, which the reference server profile does not advertise, so the client refuses before dialing. Co-Authored-By: Claude Opus 5.5 * test(cmd/bd): split the bd search route agreement into the embedded and proxied lanes TestSearchRoutesAgree skipped unless BEADS_TEST_EMBEDDED_DOLT=1, then ran its proxied and remote legs only under BEADS_TEST_PROXIED_SERVER=1, and its name matched neither lane's discovery (TestEmbedded* in *_embedded_test.go, TestProxiedServer*/TestServerMode*). No CI lane ran it, which main's #7468 guard (TestProxiedGatedTestsRunInTheProxiedLane) now fails on. Split it: - TestEmbeddedSearchRoutesAgree (search_routes_embedded_test.go): the direct route, embedded lane. - TestProxiedServerSearchRoutesAgree (search_routes_proxied_test.go): the proxied route and the remote route (bd serve over the proxied workspace), gated only on BEADS_TEST_PROXIED_SERVER. The remote route must still answer the proxied one row for row, ids and order included, and refuse the flag listIssues cannot carry. Neither lane can run the other's workspace, so the shared helper (search_routes_parity_test.go: the seed, the twelve flag sets, and checkSearchRoute) holds every route to one expected answer per flag set; agreeing with it is agreeing with each other. The remote leg's "needs S6c" skip is gone: S6c is on main, so a refusal now fails. Both manifests regenerated incrementally (embedded 100-shard block, shard 71; proxied 34-shard block, shard 23). Bazel: bd_embedded_part2_test (--config=embedded) and bd_proxied_test (--config=doltserver-proxied) pass and run the new tests; //scripts:go_test_sources_test passes. Swapping the title-sort expectation fails both routes of the proxied test. Co-Authored-By: Claude Opus 5.5 * fix(search): keep an infra type's durable rows, expand --type aliases, fail the S26 e2e on refusal (bd-1ejjv6) Review fixes for #7528, attempt 1: the three majors and three minors of the synthesis. `bd search --type ` dropped the type's durable rows (a promoted wisp, an issue retyped after it was created), exit 0. The shared listing (workapi.BuildListFilter) narrowed an infra --type to its ephemeral rows unconditionally, so SearchListRequest's IncludeEphemeral did not bring back what raw SearchIssues used to return. The narrowing now applies only without IncludeEphemeral, in BuildListFilter and in the count lockstep (applyCountIncludeInfra), and the http parent-walk recognizer reads an infra type's Ephemeral back as the include_ephemeral intent. This also fixes `bd list --type --include-ephemeral` and GET /v0/beads/issues, which shipped the narrowing in v1.3.1. Pinned by workapi tests and by template, gate, wisp and promoted-message rows in the Reader conformance Query case (new ReaderFixture.PromoteWisp) on the dolt, embedded, uow and served backends; the issueops Reader/Counter docs and the openapi descriptions say the same. `bd search --type` handed the raw string to the Reader: an alias (feat, mol, mr, ...) answered nothing and an unknown type an empty result. It is now expanded with utils.NormalizeIssueType as `bd list` does, and an unknown type exits 1. The help text and the CHANGELOG say so, and the CHANGELOG also records the additive comments_omitted field in `bd search --json` rows. TestE2E_SearchOverHTTP skipped on the very SearchIssuesWithCounts refusal it exists to catch; it now fails. The route agreement seeds "kinded" template, gate, wisp and promoted-message rows and pins the alias, the unknown-type refusal and the infra rows on the direct, proxied and remote routes. Minors: the CHANGELOG states the proxied `bd show --current` miss fallback; the CommentsOmitted contract names `bd search --json`; the storage.RecomputeBlocked and `bd recompute-blocked` comments name the deliberate --proxied-server bypass. Validation: go vet and gofmt on every changed package; make api-check; workapi, issueops, httpclient/encode, types and storage unit tests; the Reader conformance on dolt, embedded, uow and served; the embedded and proxied route agreements (remote included), the embedded search tests, TestProxiedServerSearch, TestProxiedServerRecomputeBlocked, the http e2e search/show/recompute tests; docs flag, freshness and docsync checks; and a hand-run bd search/list/count against a scratch workspace. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .../scripts/embedded-storage-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + .golangci.yml | 6 +- CHANGELOG.md | 37 ++ README.md | 9 + backend/conformance/reader_contract.go | 110 +++++ backend/conformance/role_bundle_cases.go | 1 + backend/http/BUILD.bazel | 3 + backend/http/corpus/testdata/corpus.yaml | 4 +- backend/http/e2e_recompute_blocked_test.go | 46 ++ backend/http/e2e_search_test.go | 57 +++ backend/http/e2e_show_current_test.go | 88 ++++ cmd/bd/BUILD.bazel | 5 +- cmd/bd/list_show_filter_modes.go | 50 -- cmd/bd/recompute_blocked.go | 45 +- cmd/bd/search.go | 444 +++++++----------- cmd/bd/search_proxied_server.go | 225 --------- cmd/bd/search_routes_embedded_test.go | 30 ++ cmd/bd/search_routes_parity_test.go | 232 +++++++++ cmd/bd/search_routes_proxied_test.go | 66 +++ cmd/bd/show.go | 5 +- cmd/bd/show_current.go | 52 ++ cmd/bd/show_current_test.go | 164 ++----- cmd/bd/show_proxied_server.go | 16 +- internal/httpapi/apigen/types.gen.go | 15 +- internal/httpapi/reads.go | 3 + internal/httpapi/reads_test.go | 41 +- internal/httpapi/routes.go | 15 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 47 +- .../httpapi/wireshape/testdata/golden.json | 8 + internal/httpclient/BUILD.bazel | 2 + internal/httpclient/blocked_recomputer.go | 12 + internal/httpclient/encode/encode.go | 1 + internal/httpclient/encode/parentwalk.go | 21 +- internal/httpclient/encode/parentwalk_test.go | 26 + internal/httpclient/encode/roundtrip_test.go | 1 + internal/httpclient/encode/table.go | 4 + internal/httpclient/list_search_test.go | 67 +++ internal/httpclient/role_reader.go | 28 ++ internal/httpclient/served_reader_test.go | 8 + internal/httpclient/served_test.go | 4 + internal/httpclient/wire/handshake.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 9 +- internal/storage/BUILD.bazel | 2 + internal/storage/blocked_recomputer.go | 70 +++ internal/storage/blocked_recomputer_test.go | 61 +++ internal/storage/dolt/reader_contract_test.go | 7 + .../embeddeddolt/reader_contract_test.go | 7 + internal/storage/uow/issue_reader.go | 2 +- internal/storage/uow/reader_contract_test.go | 8 + internal/types/types.go | 3 +- internal/workapi/count.go | 11 +- internal/workapi/count_test.go | 113 +++-- internal/workapi/list.go | 32 +- internal/workapi/list_ephemeral_test.go | 44 ++ internal/workapi/storereader/reader.go | 2 +- issueops/BUILD.bazel | 5 + issueops/counter.go | 8 +- issueops/current.go | 64 +++ issueops/current_test.go | 154 ++++++ issueops/reader.go | 19 + issueops/reader_ready_scope.go | 1 + issueops/search.go | 33 ++ issueops/search_test.go | 22 + 66 files changed, 1926 insertions(+), 765 deletions(-) create mode 100644 backend/http/e2e_recompute_blocked_test.go create mode 100644 backend/http/e2e_search_test.go create mode 100644 backend/http/e2e_show_current_test.go delete mode 100644 cmd/bd/search_proxied_server.go create mode 100644 cmd/bd/search_routes_embedded_test.go create mode 100644 cmd/bd/search_routes_parity_test.go create mode 100644 cmd/bd/search_routes_proxied_test.go create mode 100644 cmd/bd/show_current.go create mode 100644 internal/httpclient/blocked_recomputer.go create mode 100644 internal/httpclient/list_search_test.go create mode 100644 internal/storage/blocked_recomputer.go create mode 100644 internal/storage/blocked_recomputer_test.go create mode 100644 issueops/current.go create mode 100644 issueops/current_test.go create mode 100644 issueops/search.go create mode 100644 issueops/search_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 0ac8f3440..ed764b6b4 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -217,6 +217,7 @@ 100 71 TestEmbeddedCloseJSONReportsMoleculeAutoClose 100 71 TestEmbeddedInitRoleRoutingExplicitDefault 100 71 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget +100 71 TestEmbeddedSearchRoutesAgree 100 72 TestEmbeddedContributorCreateMultiple 100 72 TestEmbeddedCreateWithGitRemote diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 714ee1098..a9daaa528 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -380,6 +380,7 @@ 40 24 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape 40 24 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction 40 24 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 +40 24 TestEmbeddedReaderListQueryMatchesTheTitleOrTheID 40 24 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt 40 24 TestIgnoredCursorHealResumeRespectsOperatorOverride 40 24 TestLargeBatchApplyStatementCountsClassic40_Embedded diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index a752c2b07..33611719b 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -197,6 +197,7 @@ 34 23 TestProxiedServerIfRevisionGuardUpdate 34 23 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows 34 23 TestProxiedServerMolBurn +34 23 TestProxiedServerSearchRoutesAgree 34 23 TestProxiedServerServeUpdate 34 23 TestProxiedServerUpdateConcurrentClaim 34 23 TestProxiedServerUpdateWisp diff --git a/.golangci.yml b/.golangci.yml index 238e8fabf..9e1cf64a3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -467,7 +467,11 @@ linters: # tests: false` means forbidigo never analyzes a test file at all, so that # entry asserts nothing either way and scoping it would be the churn this # comment just refused to write. - - path: '^cmd/bd/(close|duplicates|find_duplicates|gate|gc|graph|info|label|orphans|purge|search|todo|wisp|human)(_proxied_server)?\.go$' + # + # `search` LEFT this list when all three of its routes moved onto + # issueops.Reader.List with issueops.SearchListRequest (cmd/bd/search.go); + # it builds no filter any more. + - path: '^cmd/bd/(close|duplicates|find_duplicates|gate|gc|graph|info|label|orphans|purge|todo|wisp|human)(_proxied_server)?\.go$' text: 'do not write a filter here' linters: - forbidigo diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6841359..83a610fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 description and metadata are unchanged, and omitting the flag (or passing `0`) is byte-identical to today's output. JSON output is untouched; `--include-comments` still streams every comment there. +- `GET /v0/beads/issues` accepts `q`, `bd search`'s free text, behind the + new `issues.list.search` capability token (additive; no wire_revision + bump). `issueops.ListRequest` gains `Query`, with `SearchIssues`' meaning + (title or id contains the text, case-insensitively), on every Reader + implementation, and `issueops.SearchListRequest` is search's scope as a + request. Every `bd search` route (direct, `--proxied-server` and the HTTP + backend) now asks `Reader.List` with it through one flag mapping; against + a server that does not advertise the token the HTTP client refuses before + dialing with a typed capability error. Two things a script can see follow + from that mapping: `bd search --type` expands aliases (`feat`, `mol`, + `dec`, ...) and validates the type as `bd list` does, so an unknown type + now exits 1 where it used to answer an empty result, and `bd search + --json` rows carry `comments_omitted` (additive) as `bd list --json` rows + do. - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's @@ -494,6 +508,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 rollout) still stores P0, with no error on either side. Out-of-tree storage backends must now honor `DefaultPriority`; see the BREAKING (out-of-tree storage backends) entry under `### Changed`. + +- `bd show --current` over the HTTP backend no longer prints the wrong issue. + Its in-progress/hooked lookup used the raw `SearchIssues` method, which + the HTTP backend refuses; the refusal was swallowed and the command fell + back to the last-touched issue, exit 0. The lookup now reads through the + `issueops.Querier` role, and a failed read is an error on every route + (including `--proxied-server`), never a fallback. A miss (nothing in + progress or hooked, or no actor) is another matter: `--proxied-server` now + falls back to the last-touched issue as the direct route always has, where + it used to fail with `no current issue found`. +- `bd list --type --include-ephemeral` (and `GET + /v0/beads/issues` with the same parameters) now include that type's durable + rows — a promoted wisp, an issue retyped after it was created — beside its + wisps, as `--include-ephemeral` promises; the type used to narrow the read + to its wisps alone. `bd count --include-infra --include-ephemeral` and the + count endpoint count the same rows. An infra `--type` without + `--include-ephemeral` still reads only the wisps, and `bd search --type + `, now served by the same listing, still finds that type's + wisps and durable rows alike. +- `bd recompute-blocked` on a workspace connected to `bd serve` is a no-op + that exits 0 instead of refusing: the server maintains `is_blocked` on + every write and the column lives in its database. `--json` reports + `{"rows_corrected": 0, "maintained_by": "server"}`. - **The http store's `GetIssue` answers a miss with `storage.ErrNotFound`, like the local stores.** It used to answer a missing (or empty) id with `(nil, nil)`, which every caller written against the backend contract diff --git a/README.md b/README.md index 6d6ef6cd7..058b949a7 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,15 @@ credential would cross the network unencrypted) — pass `--allow-plaintext` to override, or use `https://`. Switching a workspace that already selects a different backend needs `--force`. +`bd search`, `bd show --current` and `bd recompute-blocked` work over HTTP +too, each in its own way. `bd search` asks the server's listing with its text (`q`, behind the `issues.list.search` capability) and +refuses before dialing against a server that does not advertise it. +`bd show --current` finds the actor's in-progress or hooked issue through the +server's boolean query. `bd recompute-blocked` is a no-op that exits 0: the +server maintains `is_blocked` on every write, and repairing a stale flag is +done in the server's own workspace (`--json` reports +`{"rows_corrected": 0, "maintained_by": "server"}`). + An embedding program that links beads as a library rather than running `cmd/bd` registers this backend itself through the public [`backend/http`](backend/http) package (`bdhttp.Register`), and can open a diff --git a/backend/conformance/reader_contract.go b/backend/conformance/reader_contract.go index 6910a5bbb..d19fea047 100644 --- a/backend/conformance/reader_contract.go +++ b/backend/conformance/reader_contract.go @@ -113,6 +113,10 @@ type ReaderFixture struct { AddDependency func(context.Context, *types.Dependency, string) error // AddComment puts one comment on a durable issue. AddComment func(ctx context.Context, issueID, author, text string) error + // PromoteWisp moves a seeded wisp into the durable plane, as `bd promote` + // does. It is the only way to seed a DURABLE row of an infra type: creating + // one routes it to the wisps plane on every backend. + PromoteWisp func(ctx context.Context, id, actor string) error } // RunReaderReadyDefaultTypeExclusionsYieldToAnExplicitType pins @@ -786,6 +790,112 @@ func RunReaderListReadyFlagRefusesAFilterItCannotCarry(t *testing.T, ctx context assertReaderPageIDs(t, "List --id without --ready", page, []string{one}) } +// RunReaderListQueryMatchesTheTitleOrTheID pins ListRequest.Query: `bd +// search`'s text, which matches a row whose title OR id contains it, case +// insensitively, and which narrows without lifting a default exclusion — and +// issueops.SearchListRequest, the request every `bd search` route sends, which +// does lift them. +// +// EVERY KIND OF ROW the search scope promises is seeded with the text, because +// a scope that silently loses one kind answers every other row correctly: a +// template, a gate, a wisp of an infra type, and a DURABLE row of that type — +// a promoted wisp, which a type filter that narrows an infra type to its +// ephemeral rows drops with no error. Each must come back under the bare scope +// and under its own type. +// +// It scopes itself with a label rather than an id set, so the served leg runs +// it as written: listIssues publishes no id parameter. +func RunReaderListQueryMatchesTheTitleOrTheID(t *testing.T, ctx context.Context, fixture ReaderFixture) { + t.Helper() + if fixture.PromoteWisp == nil { + t.Fatal("fixture has no PromoteWisp: a durable row of an infra type cannot be seeded, and it is the row a narrowed type filter drops") + } + scope := readerLabel(fixture, "lsquery") + needle := strings.ToLower(fixture.IssuePrefix) + "qneedle" + titleHit := readerID(fixture, "lsquery", "title") + idHit := readerID(fixture, "lsquery", "idhit") + closedHit := readerID(fixture, "lsquery", "closed") + miss := readerID(fixture, "lsquery", "miss") + template := readerID(fixture, "lsquery", "template") + gate := readerID(fixture, "lsquery", "gate") + wispMessage := readerID(fixture, "lsquery", "wispmsg") + promotedMessage := readerID(fixture, "lsquery", "promotedmsg") + + // The title carries the needle in UPPER case and the request asks in lower, + // so a case-sensitive match finds nothing. + titleIssue := readerIssue(titleHit, types.TypeTask, scope) + titleIssue.Title = "fix the " + strings.ToUpper(needle) + " path" + // The id-hit's title says nothing about it: only its id can match. + idIssue := readerIssue(idHit, types.TypeTask, scope) + idIssue.Title = "unrelated words" + closedIssue := readerIssue(closedHit, types.TypeTask, scope) + closedIssue.Title = "old " + needle + closedIssue.Status = types.StatusClosed + missIssue := readerIssue(miss, types.TypeTask, scope) + missIssue.Title = "nothing to see" + templateIssue := readerIssue(template, types.TypeTask, scope) + templateIssue.Title = needle + " template" + templateIssue.IsTemplate = true + gateIssue := readerIssue(gate, types.TypeGate, scope) + gateIssue.Title = needle + " gate" + for _, issue := range []*types.Issue{titleIssue, idIssue, closedIssue, missIssue, templateIssue, gateIssue} { + seedReaderIssue(t, ctx, fixture, issue) + } + // Two rows of the same infra type, one per plane. `message` is built in, + // so the type is valid on every backend without configuration. + for _, id := range []string{wispMessage, promotedMessage} { + message := readerIssue(id, types.TypeMessage, scope) + message.Title = needle + " message" + message.Ephemeral = true + seedReaderWisp(t, ctx, fixture, message) + } + if err := fixture.PromoteWisp(ctx, promotedMessage, "seed"); err != nil { + t.Fatalf("promote wisp %s: %v", promotedMessage, err) + } + + searchType := func(issueType types.IssueType) publicops.ListRequest { + req := searchScoped(needle, scope) + req.IssueType = string(issueType) + return req + } + + for _, tc := range []struct { + name string + req publicops.ListRequest + want []string + }{ + {"a title match, case-insensitively, under the default exclusions", publicops.ListRequest{Labels: []string{scope}, Query: needle}, []string{titleHit}}, + {"AllFlag takes the closed match back", publicops.ListRequest{Labels: []string{scope}, Query: needle, AllFlag: true}, []string{titleHit, closedHit}}, + {"an id match whose title does not contain the text", publicops.ListRequest{Labels: []string{scope}, Query: idHit, AllFlag: true}, []string{idHit}}, + {"text nothing contains", publicops.ListRequest{Labels: []string{scope}, Query: needle + "-absent", AllFlag: true}, nil}, + {"no text narrows nothing", publicops.ListRequest{Labels: []string{scope}}, []string{titleHit, idHit, miss}}, + {"SearchListRequest, `bd search`'s scope, takes every status and every kind of row back", searchScoped(needle, scope), []string{titleHit, closedHit, template, gate, wispMessage, promotedMessage}}, + {"SearchListRequest under an infra type keeps both planes' rows of it", searchType(types.TypeMessage), []string{wispMessage, promotedMessage}}, + {"SearchListRequest under the gate type", searchType(types.TypeGate), []string{gate}}, + {"SearchListRequest under a plain type keeps its templates", searchType(types.TypeTask), []string{titleHit, closedHit, template}}, + // Outside the search scope an infra type alone reads only its + // ephemeral rows, which is where creating one puts it; IncludeEphemeral + // is what lifts that narrowing to the durable rows as well. + {"an infra type alone reads only its ephemeral rows", publicops.ListRequest{Labels: []string{scope}, Query: needle, IssueType: string(types.TypeMessage)}, []string{wispMessage}}, + {"IncludeEphemeral lifts that narrowing", publicops.ListRequest{Labels: []string{scope}, Query: needle, IssueType: string(types.TypeMessage), IncludeEphemeral: true}, []string{wispMessage, promotedMessage}}, + } { + page, err := fixture.Reader.List(ctx, tc.req) + if err != nil { + t.Errorf("List %s: %v", tc.name, err) + continue + } + assertReaderPageIDSet(t, "List "+tc.name, page, tc.want) + } +} + +// searchScoped is the library's search request narrowed to one case's label, +// the request every `bd search` route sends. +func searchScoped(query, label string) publicops.ListRequest { + req := publicops.SearchListRequest(query) + req.Labels = []string{label} + return req +} + // RunReaderListEmptyPageIsWellFormed pins the page shape when nothing matches // (reader.go:376-381). Items is never nil for a successful call, so no caller // has to tell null from empty to learn that nothing matched, and an empty page diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 36c158477..3453eedaa 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -568,6 +568,7 @@ var roleContractCases = []roleContract{ RunReaderListKeysetPositionResumesTheCreatedDescIDAscOrder, RunReaderListReadyFlagAnswersTheBlockerAwareSet, RunReaderListReadyFlagRefusesAFilterItCannotCarry, + RunReaderListQueryMatchesTheTitleOrTheID, RunReaderListEmptyPageIsWellFormed, RunReaderListMaxRowsIsHonored, RunReaderListMaxRowsBoundaryIsLimitPlusOffset, diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index d8d4e273a..16ea2bf83 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -33,7 +33,10 @@ go_test( "attach_test.go", "e2e_cli_test.go", "e2e_mol_test.go", + "e2e_recompute_blocked_test.go", "e2e_remote_roles_test.go", + "e2e_search_test.go", + "e2e_show_current_test.go", "open_test.go", "per_open_credential_test.go", "register_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index e7a74b0d7..ee970ef53 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -157,7 +157,6 @@ shapes: - id: gc-show-current source: gc cmd_bd_show_watch.go bdWatchRunner (show --current --json) args: [show, --current, --json] - pending: {slice: "S6e", reason: "NEW: resolveCurrentIssueID calls raw store.SearchIssues, the http refusal is swallowed, and it silently falls back to the last-touched issue (wrong issue, exit 0)"} - id: gc-show-missing source: gc BdStore.Get not-found classification (routing fallbacks) args: [show, --json, e2e-nonexistent] @@ -326,7 +325,7 @@ shapes: source: mc mol-bug-report-flow-v2 (bd search) args: [search, "Routed", --json] compare: ids - pending: {slice: "S24", reason: "NEW: bd search is refused over http ('partial-id search is not available over HTTP'); not in B3"} + pending: {slice: "S6c+SRV-1", reason: "S26 sends bd search as listIssues q behind issues.list.search, which the reference server does not advertise, so the client refuses before dialing; remove the mark when reference_server.json is upgraded"} - id: mc-status source: mc normalized_child_launch.py (bd status) args: [status, --json] @@ -412,7 +411,6 @@ shapes: source: gc bdstore_blocked_repair.go RecomputeBlocked args: [recompute-blocked, --json] compare: exit - pending: {slice: "S25", reason: "NEW: recompute-blocked refuses over http ('storage backend does not support is_blocked recompute'); gc BdStore.RecomputeBlocked issues it; not in B3"} - id: gc-backup-status source: gc checks_proxied_backup_coverage.go (backup status --json) args: [backup, status, --json] diff --git a/backend/http/e2e_recompute_blocked_test.go b/backend/http/e2e_recompute_blocked_test.go new file mode 100644 index 000000000..11d49b334 --- /dev/null +++ b/backend/http/e2e_recompute_blocked_test.go @@ -0,0 +1,46 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestE2E_RecomputeBlockedOverHTTP is S25. On a remote backend +// `bd recompute-blocked` is a no-op that exits 0: the server maintains +// is_blocked on every write and the column lives in its database. --json still +// carries rows_corrected (gc parses it) and says the server maintains the flag. +func TestE2E_RecomputeBlockedOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + // A blocked edge exists, so there is state the server derived; the client + // must not claim to have repaired or refused it. + blocked := createE2EIssue(t, bd, "blocked") + blocker := createE2EIssue(t, bd, "blocker") + mustRun(t, bd, "dep", "add", blocked, blocker, "--json") + + r := mustRun(t, bd, "recompute-blocked", "--json") + var body struct { + RowsCorrected *int `json:"rows_corrected"` + MaintainedBy string `json:"maintained_by"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(r.stdout)), &body); err != nil { + t.Fatalf("parse recompute-blocked --json: %v\n%s", err, r.stdout) + } + if body.RowsCorrected == nil || *body.RowsCorrected != 0 || body.MaintainedBy != "server" { + t.Fatalf("recompute-blocked --json = %s, want rows_corrected 0 and maintained_by server", r.stdout) + } + + text := mustRun(t, bd, "recompute-blocked") + if !strings.Contains(text.stdout, "maintained by the bd serve backend") { + t.Errorf("recompute-blocked text does not say the server maintains the flag: %s", text.stdout) + } + + // The ready set still reflects the edge: nothing was rewritten. + ready := jsonIDSet(t, mustRun(t, bd, "ready", "--json").stdout) + if ready[blocked] || !ready[blocker] { + t.Errorf("bd ready after recompute-blocked = %v, want %s ready and %s blocked", ready, blocker, blocked) + } +} diff --git a/backend/http/e2e_search_test.go b/backend/http/e2e_search_test.go new file mode 100644 index 000000000..b932fc71a --- /dev/null +++ b/backend/http/e2e_search_test.go @@ -0,0 +1,57 @@ +//go:build cgo + +package bdhttp_test + +import ( + "testing" +) + +// TestE2E_SearchOverHTTP is S26. `bd search` called the raw SearchIssues +// method, which the http backend refuses ("partial-id search is not available +// over HTTP"); it now asks Reader.List with the text in ListRequest.Query, +// which travels as listIssues' `q` behind issues.list.search. +func TestE2E_SearchOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + routed := createE2EIssue(t, bd, "Routed pool work") + alsoRouted := createE2EIssue(t, bd, "re-routed later") + closedRouted := createE2EIssue(t, bd, "routed and done") + other := createE2EIssue(t, bd, "something else") + mustRun(t, bd, "close", closedRouted, "--json") + + // A refusal of the raw SearchIssuesWithCounts seam is the S26 regression + // itself, not a missing dependency: S6c (the externaldeps reader + // passthrough) is underneath, so nothing on this route reaches that seam. + r := bd("search", "ROUTED", "--json") + if r.code != 0 { + t.Fatalf("bd search failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + got := jsonIDSet(t, r.stdout) + // Case-insensitive, every status (search includes closed by default). + for _, id := range []string{routed, alsoRouted, closedRouted} { + if !got[id] { + t.Errorf("bd search ROUTED missed %s: %s", id, r.stdout) + } + } + if got[other] { + t.Errorf("bd search ROUTED matched %s, whose title does not contain it: %s", other, r.stdout) + } + + // --status narrows, as on a local workspace. + open := jsonIDSet(t, mustRun(t, bd, "search", "routed", "--status", "open", "--json").stdout) + if open[closedRouted] || !open[routed] { + t.Errorf("bd search routed --status open = %v, want %s without %s", open, routed, closedRouted) + } + + // An id matches even when the title does not contain it. + byID := jsonIDSet(t, mustRun(t, bd, "search", other, "--json").stdout) + if !byID[other] { + t.Errorf("bd search %s did not match the issue by its id: %v", other, byID) + } + + // A flag listIssues publishes nothing for refuses rather than widening the + // search. + if r := bd("search", "routed", "--desc-contains", "pool", "--json"); r.code == 0 { + t.Errorf("bd search --desc-contains succeeded over http; the wire cannot carry it, so it must refuse: %s", r.stdout) + } +} diff --git a/backend/http/e2e_show_current_test.go b/backend/http/e2e_show_current_test.go new file mode 100644 index 000000000..207f42756 --- /dev/null +++ b/backend/http/e2e_show_current_test.go @@ -0,0 +1,88 @@ +//go:build cgo + +package bdhttp_test + +import ( + "strings" + "testing" +) + +// connectE2EWorkspace starts an in-process server and returns a workspace +// `bd connect`ed to it, plus a runner that drives the built binary there as +// the given actor. +func connectE2EWorkspace(t *testing.T, actor string) func(args ...string) bdResult { + t.Helper() + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + workspace := t.TempDir() + env := []string{"BEADS_ACTOR=" + actor} + if r := runBD(t, bin, workspace, env, "connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json"); r.code != 0 { + t.Fatalf("bd connect failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + return func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, workspace, env, args...) + } +} + +func mustRun(t *testing.T, bd func(args ...string) bdResult, args ...string) bdResult { + t.Helper() + r := bd(args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + return r +} + +func createE2EIssue(t *testing.T, bd func(args ...string) bdResult, title string) string { + t.Helper() + id := firstJSONField(t, mustRun(t, bd, "create", title, "--json").stdout, "id") + if id == "" { + t.Fatalf("bd create %q --json produced no id", title) + } + return id +} + +// TestE2E_ShowCurrentOverHTTP is S6e. `bd show --current` used to read the +// actor's in-progress work through the raw SearchIssues method, which the http +// backend refuses; the refusal was swallowed and the command printed the +// LAST-TOUCHED issue instead, exit 0. It now reads through the Querier role +// (queryIssues), so the in-progress issue wins over a more recently touched +// one, and a hooked issue is found when nothing is in progress. +func TestE2E_ShowCurrentOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + current := createE2EIssue(t, bd, "the work in progress") + hooked := createE2EIssue(t, bd, "hooked work") + touched := createE2EIssue(t, bd, "touched last") + + mustRun(t, bd, "update", current, "--status", "in_progress", "--assignee", "worker", "--json") + // Touch another issue AFTER claiming, so last-touched is not the current + // issue: the old swallowed refusal answered with exactly this one. + mustRun(t, bd, "update", touched, "--priority", "1", "--json") + + showCurrent := func() string { + t.Helper() + r := mustRun(t, bd, "show", "--current", "--json") + return firstJSONField(t, r.stdout, "id") + } + if got := showCurrent(); got != current { + t.Fatalf("bd show --current = %s, want the in-progress %s (last touched was %s)", got, current, touched) + } + + // Nothing in progress: the hooked issue assigned to the actor is current. + mustRun(t, bd, "update", current, "--status", "open", "--assignee", "", "--json") + mustRun(t, bd, "update", hooked, "--status", "hooked", "--assignee", "worker", "--json") + mustRun(t, bd, "update", touched, "--priority", "2", "--json") + if got := showCurrent(); got != hooked { + t.Fatalf("bd show --current = %s, want the hooked %s (last touched was %s)", got, hooked, touched) + } + + // Neither: last-touched is the answer, and only now. + mustRun(t, bd, "update", hooked, "--status", "open", "--assignee", "", "--json") + mustRun(t, bd, "update", touched, "--priority", "3", "--json") + if got := showCurrent(); got != touched { + t.Fatalf("bd show --current = %s, want the last-touched %s when nothing is in progress or hooked", got, touched) + } +} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index bcf823bf2..9234892a4 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -314,13 +314,13 @@ go_library( "schema.go", "schema_skew.go", "search.go", - "search_proxied_server.go", "send_metrics.go", "serve.go", "setup.go", "ship.go", "show.go", "show_children.go", + "show_current.go", "show_display.go", "show_format.go", "show_proxied_server.go", @@ -1112,6 +1112,9 @@ go_test( "schema_test.go", "search_embedded_test.go", "search_proxied_integration_test.go", + "search_routes_embedded_test.go", + "search_routes_parity_test.go", + "search_routes_proxied_test.go", "search_test.go", "send_metrics_diagnostics_test.go", "serve_batch_close_proxied_integration_test.go", diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index ef67625f8..08c40f1ac 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -298,53 +298,3 @@ func gatherProxiedHierarchical(ctx context.Context, uw uow.UnitOfWork, parentID out = append(out, descendants...) return out, nil } - -type currentIssueSearcher interface { - SearchIssues(context.Context, string, types.IssueFilter) ([]*types.Issue, error) -} - -// resolveCurrentIssueID determines the current active issue for the agent. -// Priority: in-progress assigned to actor > hooked > last touched. -func resolveCurrentIssueID(ctx context.Context) string { - return resolveCurrentIssueIDFrom(ctx, store, getActorWithGit, GetLastTouchedID) -} - -func resolveCurrentIssueIDFrom(ctx context.Context, searcher currentIssueSearcher, currentActor func() string, fallback func() string) string { - if searcher == nil { - return fallback() - } - actor := currentActor() - if actor == "" { - return fallback() - } - - for _, status := range []types.Status{types.StatusInProgress, types.StatusHooked} { - status := status - filter := types.IssueFilter{ - Status: &status, - Assignee: &actor, - } - issues, err := searcher.SearchIssues(ctx, "", filter) - if err == nil && len(issues) > 0 { - return issues[0].ID - } - } - - return fallback() -} - -func resolveCurrentIssueIDProxied(ctx context.Context, uw uow.UnitOfWork) string { - currentActor := getActorWithGit() - if currentActor == "" { - return "" - } - for _, status := range []types.Status{types.StatusInProgress, types.StatusHooked} { - st := status - filter := types.IssueFilter{Status: &st, Assignee: ¤tActor} - page, err := uw.IssueUseCase().SearchIssues(ctx, "", filter) - if err == nil && len(page.Items) > 0 { - return page.Items[0].ID - } - } - return "" -} diff --git a/cmd/bd/recompute_blocked.go b/cmd/bd/recompute_blocked.go index 050495126..2d7fe6692 100644 --- a/cmd/bd/recompute_blocked.go +++ b/cmd/bd/recompute_blocked.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "github.com/spf13/cobra" @@ -27,6 +28,11 @@ It is idempotent: on a consistent database it changes nothing. Works in every storage mode — embedded, server, and proxied-server (unlike 'bd doctor', which is server-mode only). +On a workspace connected to a bd serve backend ('bd connect') it is a no-op +that exits 0: the server maintains is_blocked on every write, and the column +lives in the server's database, so the repair is run there, by the server's +operator. --json then reports {"rows_corrected": 0, "maintained_by": "server"}. + Examples: bd recompute-blocked # Repair stale is_blocked flags bd recompute-blocked --json # Machine-parseable {"rows_corrected": N}`, @@ -48,15 +54,23 @@ Examples: return runRecomputeBlockedProxiedServer(ctx) } - recomputer, ok := storage.UnwrapStore(store).(storage.BlockedRecomputer) - if !ok { - return HandleError("storage backend does not support is_blocked recompute") - } - changed, err := recomputer.RecomputeAllBlocked(ctx) + // One library entry for every store (storage.RecomputeBlocked): a + // local store recomputes its own column; a remote backend (`bd + // connect`) answers that its server maintains the column, and + // recomputes nothing. The --proxied-server route above bypasses it on + // purpose; runRecomputeBlockedProxiedServer says why. + result, err := storage.RecomputeBlocked(ctx, store) if err != nil { + var unsup *storage.ErrUnsupported + if errors.As(err, &unsup) { + return HandleError("storage backend does not support is_blocked recompute") + } return HandleError("recompute is_blocked: %v", err) } - return renderRecomputeBlocked(changed) + if result.MaintainedBy == storage.BlockedMaintainedByServer { + return renderRecomputeBlockedServerMaintained() + } + return renderRecomputeBlocked(result.RowsCorrected) }, } @@ -76,6 +90,25 @@ func renderRecomputeBlocked(changed int) error { return nil } +// renderRecomputeBlockedServerMaintained renders a remote backend's answer +// (storage.BlockedMaintainedByServer): nothing was recomputed here, because +// the server derives the column and it lives in the server's database — see +// storage.RecomputeBlocked for why that is a no-op rather than a wire +// operation. +// +// It still answers rows_corrected (0) because callers parse it (gc's +// BdStore.RecomputeBlocked requires the member, and wh-bridge-sync reads the +// exit code), and it carries `maintained_by: "server"` so a caller that cares +// can tell "nothing was stale" from "nothing was checked here". +func renderRecomputeBlockedServerMaintained() error { + if jsonOutput { + return outputJSON(map[string]interface{}{"rows_corrected": 0, "maintained_by": storage.BlockedMaintainedByServer}) + } + fmt.Println("is_blocked is maintained by the bd serve backend on every write; nothing to recompute from this client.") + fmt.Println("To repair a stale flag on the server's database, run bd recompute-blocked in the server's workspace.") + return nil +} + func init() { rootCmd.AddCommand(recomputeBlockedCmd) } diff --git a/cmd/bd/search.go b/cmd/bd/search.go index 80dbe8ff7..f12a5b580 100644 --- a/cmd/bd/search.go +++ b/cmd/bd/search.go @@ -1,9 +1,11 @@ package main import ( + "errors" "fmt" "os" "strings" + "time" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" @@ -12,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/internal/validation" "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" ) var searchCmd = &cobra.Command{ @@ -49,273 +52,184 @@ Examples: c.CloseEventAndAdd(evt) } }() - - if usesProxiedServer() { - return runSearchProxiedServer(cmd, rootCtx, args) - } - - queryFlag, _ := cmd.Flags().GetString("query") - var query string - if len(args) > 0 { - query = strings.Join(args, " ") - } else if queryFlag != "" { - query = queryFlag - } - - if query == "" { - if err := cmd.Help(); err != nil { - fmt.Fprintf(os.Stderr, "Error displaying help: %v\n", err) - } - return HandleError("search query is required") - } - - // Get filter flags - status, _ := cmd.Flags().GetString("status") - assignee, _ := cmd.Flags().GetString("assignee") - issueType, _ := cmd.Flags().GetString("type") - limit, _ := cmd.Flags().GetInt("limit") - labels, _ := cmd.Flags().GetStringSlice("label") - labelsAny, _ := cmd.Flags().GetStringSlice("label-any") - longFormat, _ := cmd.Flags().GetBool("long") - sortBy, _ := cmd.Flags().GetString("sort") - reverse, _ := cmd.Flags().GetBool("reverse") - - // Date range flags - createdAfter, _ := cmd.Flags().GetString("created-after") - createdBefore, _ := cmd.Flags().GetString("created-before") - updatedAfter, _ := cmd.Flags().GetString("updated-after") - updatedBefore, _ := cmd.Flags().GetString("updated-before") - closedAfter, _ := cmd.Flags().GetString("closed-after") - closedBefore, _ := cmd.Flags().GetString("closed-before") - - // Priority range flags - priorityMinStr, _ := cmd.Flags().GetString("priority-min") - priorityMaxStr, _ := cmd.Flags().GetString("priority-max") - - // Pattern matching flags - descContains, _ := cmd.Flags().GetString("desc-contains") - notesContains, _ := cmd.Flags().GetString("notes-contains") - externalContains, _ := cmd.Flags().GetString("external-contains") - - // Empty/null check flags - emptyDesc, _ := cmd.Flags().GetBool("empty-description") - noAssignee, _ := cmd.Flags().GetBool("no-assignee") - noLabels, _ := cmd.Flags().GetBool("no-labels") - - // Normalize labels - labels = utils.NormalizeLabels(labels) - labelsAny = utils.NormalizeLabels(labelsAny) - - // Build filter - filter := types.IssueFilter{ - Limit: limit, - } - - if status != "" && status != "all" { - cfg, err := workapi.LoadStoreListConfig(rootCtx, store) - if err != nil { - return HandleError("loading status configuration: %v", err) - } - if err := workapi.ApplyStatusFilter(&filter, status, cfg.CustomStatusNames()); err != nil { - return HandleError("%v", err) - } - } - // Default (no --status) searches ALL statuses including closed - // (bd-t5yex): the dominant real-world query is "was this already - // found/filed/fixed?" — exactly where silently excluding closed - // issues produces a false "no". This reverses the hq-319 open-only - // default, which traded that correctness for scan scope; narrow - // explicitly (e.g. --status open) when performance matters. - - if assignee != "" { - filter.Assignee = &assignee - } - - if issueType != "" { - t := types.IssueType(issueType) - filter.IssueType = &t - } - - if len(labels) > 0 { - filter.Labels = labels - } - - if len(labelsAny) > 0 { - filter.LabelsAny = labelsAny - } - - // Pattern matching - if descContains != "" { - filter.DescriptionContains = descContains - } - if notesContains != "" { - filter.NotesContains = notesContains - } - if externalContains != "" { - filter.ExternalRefContains = externalContains - } - - // Empty/null checks - if emptyDesc { - filter.EmptyDescription = true - } - if noAssignee { - filter.NoAssignee = true - } - if noLabels { - filter.NoLabels = true - } - - // Date ranges - if createdAfter != "" { - t, err := parseTimeFlag(createdAfter) - if err != nil { - return HandleError("parsing --created-after: %v", err) - } - filter.CreatedAfter = &t - } - if createdBefore != "" { - t, err := parseTimeFlag(createdBefore) - if err != nil { - return HandleError("parsing --created-before: %v", err) - } - filter.CreatedBefore = &t - } - if updatedAfter != "" { - t, err := parseTimeFlag(updatedAfter) - if err != nil { - return HandleError("parsing --updated-after: %v", err) - } - filter.UpdatedAfter = &t - } - if updatedBefore != "" { - t, err := parseTimeFlag(updatedBefore) - if err != nil { - return HandleError("parsing --updated-before: %v", err) - } - filter.UpdatedBefore = &t - } - if closedAfter != "" { - t, err := parseTimeFlag(closedAfter) - if err != nil { - return HandleError("parsing --closed-after: %v", err) - } - filter.ClosedAfter = &t - } - if closedBefore != "" { - t, err := parseTimeFlag(closedBefore) - if err != nil { - return HandleError("parsing --closed-before: %v", err) - } - filter.ClosedBefore = &t - } - - if cmd.Flags().Changed("priority-min") { - priorityMin, err := validation.ValidatePriority(priorityMinStr) - if err != nil { - return HandleError("parsing --priority-min: %v", err) - } - filter.PriorityMin = &priorityMin - } - if cmd.Flags().Changed("priority-max") { - priorityMax, err := validation.ValidatePriority(priorityMaxStr) - if err != nil { - return HandleError("parsing --priority-max: %v", err) - } - filter.PriorityMax = &priorityMax - } - - metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") - if len(metadataFieldFlags) > 0 { - filter.MetadataFields = make(map[string]string, len(metadataFieldFlags)) - for _, mf := range metadataFieldFlags { - k, v, ok := strings.Cut(mf, "=") - if !ok || k == "" { - return HandleError("invalid --metadata-field: expected key=value, got %q", mf) - } - if err := storage.ValidateMetadataKey(k); err != nil { - return HandleError("invalid --metadata-field key: %v", err) - } - filter.MetadataFields[k] = v - } - } - hasMetadataKey, _ := cmd.Flags().GetString("has-metadata-key") - if hasMetadataKey != "" { - if err := storage.ValidateMetadataKey(hasMetadataKey); err != nil { - return HandleError("invalid --has-metadata-key: %v", err) - } - filter.HasMetadataKey = hasMetadataKey - } - - ctx := rootCtx - - issues, err := store.SearchIssues(ctx, query, filter) - if err != nil { - return HandleError("%v", err) - } - - // Apply sorting - workapi.SortIssues(issues, sortBy, reverse) - - if jsonOutput { - // Get labels and dependency counts - issueIDs := make([]string, len(issues)) - for i, issue := range issues { - issueIDs[i] = issue.ID - } - labelsMap, err := store.GetLabelsForIssues(ctx, issueIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to get labels: %v\n", err) - labelsMap = make(map[string][]string) - } - depCounts, err := store.GetDependencyCounts(ctx, issueIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to get dependency counts: %v\n", err) - depCounts = make(map[string]*types.DependencyCounts) - } - commentCounts, err := store.GetCommentCounts(ctx, issueIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to get comment counts: %v\n", err) - commentCounts = make(map[string]int) - } - - // Populate labels - for _, issue := range issues { - issue.Labels = labelsMap[issue.ID] - } - - // Build response with counts - issuesWithCounts := make([]*types.IssueWithCounts, len(issues)) - for i, issue := range issues { - counts := depCounts[issue.ID] - if counts == nil { - counts = &types.DependencyCounts{DependencyCount: 0, DependentCount: 0} - } - issuesWithCounts[i] = &types.IssueWithCounts{ - Issue: issue, - DependencyCount: counts.DependencyCount, - DependentCount: counts.DependentCount, - CommentCount: commentCounts[issue.ID], - } - } - return outputJSON(issuesWithCounts) - } - - // Load labels for display - issueIDs := make([]string, len(issues)) - for i, issue := range issues { - issueIDs[i] = issue.ID - } - labelsMap, _ := store.GetLabelsForIssues(ctx, issueIDs) - for _, issue := range issues { - issue.Labels = labelsMap[issue.ID] - } - - outputSearchResults(issues, query, longFormat) - return nil + return runSearch(cmd, args) }, } +// runSearch is `bd search` on every route — the direct store, the proxied +// server and a remote backend — and every route asks the same library entry: +// issueops.Reader.List with issueops.SearchListRequest (plus the narrowing +// flags), whose ListRequest.Query is SearchIssues' free text. The route only +// decides WHICH reader answers: the store's, or the proxied provider's. A +// remote backend's reader sends the text as listIssues' `q`, which the +// server's handler hands to the same role, and refuses before dialing against +// a server that does not advertise issues.list.search. +// +// The page is the first --limit rows in the store's default order (the +// request's SortBy); --sort/--reverse reorder that page here, which is what +// this command has always done. +func runSearch(cmd *cobra.Command, args []string) error { + fail := HandleError + if usesProxiedServer() { + fail = HandleErrorRespectJSON + } + + queryFlag, _ := cmd.Flags().GetString("query") + query := queryFlag + if len(args) > 0 { + query = strings.Join(args, " ") + } + if query == "" { + if err := cmd.Help(); err != nil { + fmt.Fprintf(os.Stderr, "Error displaying help: %v\n", err) + } + return fail("search query is required") + } + + req, err := searchListRequest(cmd, query) + if err != nil { + return fail("%v", err) + } + + reader, err := searchReader() + if err != nil { + return fail("%v", err) + } + page, err := reader.List(rootCtx, req) + if err != nil { + var unsup *storage.ErrUnsupported + if errors.As(err, &unsup) && unsup.Capability != "" { + return fail("bd search needs a bd serve that advertises %s: %v", unsup.Capability, err) + } + return fail("%v", err) + } + + sortBy, _ := cmd.Flags().GetString("sort") + reverse, _ := cmd.Flags().GetBool("reverse") + + if jsonOutput { + items := make([]*types.IssueWithCounts, 0, len(page.Items)) + for _, row := range page.Items { + if row != nil && row.Issue != nil { + items = append(items, row) + } + } + workapi.SortIssuesWithCounts(items, sortBy, reverse) + return outputJSON(items) + } + + issues, _ := listPageIssues(page) + workapi.SortIssues(issues, sortBy, reverse) + longFormat, _ := cmd.Flags().GetBool("long") + outputSearchResults(issues, query, longFormat) + return nil +} + +// searchReader is the Reader this invocation's route answers through, taken +// from the route's own accessor so its decorators are in place. +func searchReader() (issueops.Reader, error) { + if usesProxiedServer() { + return proxiedIssueReader() + } + if store == nil { + return nil, errors.New("no storage available") + } + return store.IssueReader() +} + +// searchListRequest maps `bd search`'s flags onto issueops.SearchListRequest. +// Flags a backend's Reader cannot carry (over http: --updated-*, --closed-*, +// --priority-min/max, the text-contains and empty/absent filters) are put on +// the request anyway, so that Reader refuses them rather than answering a +// wider search than was asked for. +func searchListRequest(cmd *cobra.Command, query string) (issueops.ListRequest, error) { + flags := cmd.Flags() + str := func(name string) string { v, _ := flags.GetString(name); return v } + boolean := func(name string) bool { v, _ := flags.GetBool(name); return v } + limit, _ := flags.GetInt("limit") + labels, _ := flags.GetStringSlice("label") + labelsAny, _ := flags.GetStringSlice("label-any") + + req := issueops.SearchListRequest(query) + req.Limit = &limit + if status := str("status"); status != "all" { + req.Status = status + } + req.Assignee = str("assignee") + // Expanded before the Reader validates it, as `bd list` does + // (list_input.go): an alias (feat, mol, mr, ...) names its canonical type, + // and a name the workspace does not know is refused, never an empty answer. + req.IssueType = utils.NormalizeIssueType(str("type")) + req.Labels = utils.NormalizeLabels(labels) + req.LabelsAny = utils.NormalizeLabels(labelsAny) + req.DescContains = str("desc-contains") + req.NotesContains = str("notes-contains") + req.ExternalContains = str("external-contains") + req.EmptyDesc = boolean("empty-description") + req.NoAssignee = boolean("no-assignee") + req.NoLabels = boolean("no-labels") + + for _, d := range []struct { + flag string + dst **time.Time + }{ + {"created-after", &req.CreatedAfter}, + {"created-before", &req.CreatedBefore}, + {"updated-after", &req.UpdatedAfter}, + {"updated-before", &req.UpdatedBefore}, + {"closed-after", &req.ClosedAfter}, + {"closed-before", &req.ClosedBefore}, + } { + raw := str(d.flag) + if raw == "" { + continue + } + t, err := parseTimeFlag(raw) + if err != nil { + return req, fmt.Errorf("parsing --%s: %w", d.flag, err) + } + *d.dst = &t + } + + for _, p := range []struct { + flag string + dst **int + }{ + {"priority-min", &req.PriorityMin}, + {"priority-max", &req.PriorityMax}, + } { + if !flags.Changed(p.flag) { + continue + } + v, err := validation.ValidatePriority(str(p.flag)) + if err != nil { + return req, fmt.Errorf("parsing --%s: %w", p.flag, err) + } + *p.dst = &v + } + + if fields, _ := flags.GetStringArray("metadata-field"); len(fields) > 0 { + req.MetadataFields = make(map[string]string, len(fields)) + for _, mf := range fields { + k, v, ok := strings.Cut(mf, "=") + if !ok || k == "" { + return req, fmt.Errorf("invalid --metadata-field: expected key=value, got %q", mf) + } + if err := storage.ValidateMetadataKey(k); err != nil { + return req, fmt.Errorf("invalid --metadata-field key: %w", err) + } + req.MetadataFields[k] = v + } + } + if key := str("has-metadata-key"); key != "" { + if err := storage.ValidateMetadataKey(key); err != nil { + return req, fmt.Errorf("invalid --has-metadata-key: %w", err) + } + req.HasMetadataKey = key + } + return req, nil +} + // outputSearchResults formats and displays search results func outputSearchResults(issues []*types.Issue, query string, longFormat bool) { if len(issues) == 0 { @@ -360,7 +274,7 @@ func init() { searchCmd.Flags().String("query", "", "Search query (alternative to positional argument)") searchCmd.Flags().StringP("status", "s", "", "Filter by stored status (comma-separated for OR; open, in_progress, blocked, deferred, closed, all). Default searches all statuses including closed. Note: dependency-blocked issues use 'bd blocked'") searchCmd.Flags().StringP("assignee", "a", "", "Filter by assignee") - searchCmd.Flags().StringP("type", "t", "", "Filter by type (bug, feature, task, epic, chore, decision, merge-request, molecule, gate)") + searchCmd.Flags().StringP("type", "t", "", "Filter by type (bug, feature, task, epic, chore, decision, message, molecule, gate, spike, story, milestone); custom types require types.custom config, and an unknown type is an error. Aliases: feat/enhancement→feature, mol→molecule, dec/adr→decision, mr→merge-request") searchCmd.Flags().StringSliceP("label", "l", []string{}, "Filter by labels (AND: must have ALL)") searchCmd.Flags().StringSlice("label-any", []string{}, "Filter by labels (OR: must have AT LEAST ONE)") searchCmd.Flags().IntP("limit", "n", 50, "Limit results (default: 50)") diff --git a/cmd/bd/search_proxied_server.go b/cmd/bd/search_proxied_server.go deleted file mode 100644 index cf98c4074..000000000 --- a/cmd/bd/search_proxied_server.go +++ /dev/null @@ -1,225 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/types" - "github.com/steveyegge/beads/internal/utils" - "github.com/steveyegge/beads/internal/validation" - "github.com/steveyegge/beads/internal/workapi" -) - -func runSearchProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { - queryFlag, _ := cmd.Flags().GetString("query") - var query string - if len(args) > 0 { - query = strings.Join(args, " ") - } else if queryFlag != "" { - query = queryFlag - } - - if query == "" { - if err := cmd.Help(); err != nil { - fmt.Fprintf(os.Stderr, "Error displaying help: %v\n", err) - } - return HandleErrorRespectJSON("search query is required") - } - - status, _ := cmd.Flags().GetString("status") - assignee, _ := cmd.Flags().GetString("assignee") - issueType, _ := cmd.Flags().GetString("type") - limit, _ := cmd.Flags().GetInt("limit") - labels, _ := cmd.Flags().GetStringSlice("label") - labelsAny, _ := cmd.Flags().GetStringSlice("label-any") - longFormat, _ := cmd.Flags().GetBool("long") - sortBy, _ := cmd.Flags().GetString("sort") - reverse, _ := cmd.Flags().GetBool("reverse") - - createdAfter, _ := cmd.Flags().GetString("created-after") - createdBefore, _ := cmd.Flags().GetString("created-before") - updatedAfter, _ := cmd.Flags().GetString("updated-after") - updatedBefore, _ := cmd.Flags().GetString("updated-before") - closedAfter, _ := cmd.Flags().GetString("closed-after") - closedBefore, _ := cmd.Flags().GetString("closed-before") - - priorityMinStr, _ := cmd.Flags().GetString("priority-min") - priorityMaxStr, _ := cmd.Flags().GetString("priority-max") - - descContains, _ := cmd.Flags().GetString("desc-contains") - notesContains, _ := cmd.Flags().GetString("notes-contains") - externalContains, _ := cmd.Flags().GetString("external-contains") - - emptyDesc, _ := cmd.Flags().GetBool("empty-description") - noAssignee, _ := cmd.Flags().GetBool("no-assignee") - noLabels, _ := cmd.Flags().GetBool("no-labels") - - labels = utils.NormalizeLabels(labels) - labelsAny = utils.NormalizeLabels(labelsAny) - - filter := types.IssueFilter{ - Limit: limit, - } - - // Default (no --status) searches all statuses including closed - // (bd-t5yex) — keep in lockstep with the embedded path in search.go. - - if assignee != "" { - filter.Assignee = &assignee - } - - if issueType != "" { - t := types.IssueType(issueType) - filter.IssueType = &t - } - - if len(labels) > 0 { - filter.Labels = labels - } - - if len(labelsAny) > 0 { - filter.LabelsAny = labelsAny - } - - if descContains != "" { - filter.DescriptionContains = descContains - } - if notesContains != "" { - filter.NotesContains = notesContains - } - if externalContains != "" { - filter.ExternalRefContains = externalContains - } - - if emptyDesc { - filter.EmptyDescription = true - } - if noAssignee { - filter.NoAssignee = true - } - if noLabels { - filter.NoLabels = true - } - - if createdAfter != "" { - t, err := parseTimeFlag(createdAfter) - if err != nil { - return HandleErrorRespectJSON("parsing --created-after: %v", err) - } - filter.CreatedAfter = &t - } - if createdBefore != "" { - t, err := parseTimeFlag(createdBefore) - if err != nil { - return HandleErrorRespectJSON("parsing --created-before: %v", err) - } - filter.CreatedBefore = &t - } - if updatedAfter != "" { - t, err := parseTimeFlag(updatedAfter) - if err != nil { - return HandleErrorRespectJSON("parsing --updated-after: %v", err) - } - filter.UpdatedAfter = &t - } - if updatedBefore != "" { - t, err := parseTimeFlag(updatedBefore) - if err != nil { - return HandleErrorRespectJSON("parsing --updated-before: %v", err) - } - filter.UpdatedBefore = &t - } - if closedAfter != "" { - t, err := parseTimeFlag(closedAfter) - if err != nil { - return HandleErrorRespectJSON("parsing --closed-after: %v", err) - } - filter.ClosedAfter = &t - } - if closedBefore != "" { - t, err := parseTimeFlag(closedBefore) - if err != nil { - return HandleErrorRespectJSON("parsing --closed-before: %v", err) - } - filter.ClosedBefore = &t - } - - if cmd.Flags().Changed("priority-min") { - priorityMin, err := validation.ValidatePriority(priorityMinStr) - if err != nil { - return HandleErrorRespectJSON("parsing --priority-min: %v", err) - } - filter.PriorityMin = &priorityMin - } - if cmd.Flags().Changed("priority-max") { - priorityMax, err := validation.ValidatePriority(priorityMaxStr) - if err != nil { - return HandleErrorRespectJSON("parsing --priority-max: %v", err) - } - filter.PriorityMax = &priorityMax - } - - metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") - if len(metadataFieldFlags) > 0 { - filter.MetadataFields = make(map[string]string, len(metadataFieldFlags)) - for _, mf := range metadataFieldFlags { - k, v, ok := strings.Cut(mf, "=") - if !ok || k == "" { - return HandleErrorRespectJSON("invalid --metadata-field: expected key=value, got %q", mf) - } - if err := storage.ValidateMetadataKey(k); err != nil { - return HandleErrorRespectJSON("invalid --metadata-field key: %v", err) - } - filter.MetadataFields[k] = v - } - } - hasMetadataKey, _ := cmd.Flags().GetString("has-metadata-key") - if hasMetadataKey != "" { - if err := storage.ValidateMetadataKey(hasMetadataKey); err != nil { - return HandleErrorRespectJSON("invalid --has-metadata-key: %v", err) - } - filter.HasMetadataKey = hasMetadataKey - } - - uw, err := openProxiedListUOW(ctx) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - defer uw.Close(ctx) - - if status != "" && status != "all" { - cfg, err := workapi.LoadUOWListConfig(ctx, uw) - if err != nil { - return HandleErrorRespectJSON("loading status configuration: %v", err) - } - if err := workapi.ApplyStatusFilter(&filter, status, cfg.CustomStatusNames()); err != nil { - return HandleErrorRespectJSON("%v", err) - } - } - - if jsonOutput { - page, err := uw.IssueUseCase().SearchIssuesWithCounts(ctx, query, filter) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - items := page.Items - workapi.SortIssuesWithCounts(items, sortBy, reverse) - if items == nil { - items = []*types.IssueWithCounts{} - } - return outputJSON(items) - } - - page, err := uw.IssueUseCase().SearchIssues(ctx, query, filter) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - issues := page.Items - workapi.SortIssues(issues, sortBy, reverse) - outputSearchResults(issues, query, longFormat) - return nil -} diff --git a/cmd/bd/search_routes_embedded_test.go b/cmd/bd/search_routes_embedded_test.go new file mode 100644 index 000000000..df0303109 --- /dev/null +++ b/cmd/bd/search_routes_embedded_test.go @@ -0,0 +1,30 @@ +//go:build cgo + +package main + +import ( + "os" + "testing" +) + +// TestEmbeddedSearchRoutesAgree is the direct route of the `bd search` route +// agreement (search_routes_parity_test.go): an embedded workspace answers +// every flag set as searchRouteQueries expects. +// TestProxiedServerSearchRoutesAgree holds the proxied and remote routes to +// the same answers. +func TestEmbeddedSearchRoutesAgree(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "sp") + seedSearchRoutes(searchRouteOps{ + create: func(title string, args []string) string { + return bdCreate(t, bd, dir, append([]string{title}, args...)...).ID + }, + close: func(id string) { bdClose(t, bd, dir, id) }, + markTemplate: func(id string) { markIssueTemplate(t, beadsDir, "sp", id) }, + promote: func(id string) { bdPromote(t, bd, dir, id) }, + }) + checkSearchRoute(t, "direct", bd, dir, bdEnv(dir), false) +} diff --git a/cmd/bd/search_routes_parity_test.go b/cmd/bd/search_routes_parity_test.go new file mode 100644 index 000000000..af851f54d --- /dev/null +++ b/cmd/bd/search_routes_parity_test.go @@ -0,0 +1,232 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os/exec" + "slices" + "strings" + "testing" +) + +// The `bd search` route agreement. Every route (the direct store, the proxied +// server and a remote backend) builds ONE request (searchListRequest over +// issueops.SearchListRequest) and asks ONE library entry with it +// (issueops.Reader.List), so the same seed must answer the same flags the same +// way on all of them. A route that mapped a flag on its own, or reached a +// different read, would answer differently. +// +// The routes run in two lanes: TestEmbeddedSearchRoutesAgree (the direct +// route, embedded lane) and TestProxiedServerSearchRoutesAgree (the proxied +// and remote routes, proxied lane). Neither lane can run the other's +// workspace, so both hold their answers to the same expectation, +// searchRouteQueries' want, through checkSearchRoute. Agreeing with it on +// every route is agreeing with each other. + +// searchRouteSeed is one row of the seed every route gets. Distinct priorities +// among the "routed" rows make a --limit cut the same rows on every route +// regardless of creation-time ties. +// +// The "kinded" rows are one of each kind of row a listing hides by default and +// search's scope takes back (issueops.SearchListRequest): a template, a gate, +// an infra-typed wisp and a durable row of the same infra type. They match a +// word of their own rather than "routed", so the ordered queries below still +// see one row per priority. +type searchRouteSeed struct { + title string + args []string + then searchRouteStep +} + +// searchRouteStep is what a route does to a seed row once it has created it. +type searchRouteStep int + +const ( + srAsCreated searchRouteStep = iota + srClose + srMarkTemplate // no create flag makes a template, so this is raw SQL + srPromote // out of the ephemeral plane creating an infra type puts it in +) + +var searchRouteSeeds = []searchRouteSeed{ + {"Routed alpha task", []string{"--type", "task", "--priority", "0", "--assignee", "alice", "--label", "lane:a"}, srAsCreated}, + {"Routed beta bug", []string{"--type", "bug", "--priority", "1", "--assignee", "bob", "--label", "lane:b"}, srAsCreated}, + {"re-ROUTED gamma", []string{"--type", "feature", "--priority", "2", "--label", "lane:a"}, srAsCreated}, + {"Routed and done", []string{"--type", "task", "--priority", "3"}, srClose}, + {"unrelated delta", []string{"--type", "task", "--priority", "4", "--label", "lane:a"}, srAsCreated}, + {"Kinded template", []string{"--type", "task"}, srMarkTemplate}, + {"Kinded gate", []string{"--type", "gate"}, srAsCreated}, + {"Kinded wisp message", []string{"--type", "message"}, srAsCreated}, + {"Kinded promoted message", []string{"--type", "message"}, srPromote}, +} + +const ( + srAlpha = "Routed alpha task|open" + srBeta = "Routed beta bug|open" + srGamma = "re-ROUTED gamma|open" + srDone = "Routed and done|closed" + + skTemplate = "Kinded template|open" + skGate = "Kinded gate|open" + skWisp = "Kinded wisp message|open" + skPromoted = "Kinded promoted message|open" +) + +// searchRouteQuery is one flag set. want is the seed's answer as title|status: +// in output order when ordered, otherwise as a set. +type searchRouteQuery struct { + name string + args []string + ordered bool // the flags fix the output order + wireCan bool // listIssues can carry every flag + want []string +} + +var searchRouteQueries = []searchRouteQuery{ + {"text", []string{"routed"}, false, true, []string{srAlpha, srBeta, srGamma, srDone}}, + {"status open", []string{"routed", "--status", "open"}, false, true, []string{srAlpha, srBeta, srGamma}}, + {"status all", []string{"routed", "--status", "all"}, false, true, []string{srAlpha, srBeta, srGamma, srDone}}, + {"type", []string{"routed", "--type", "bug"}, false, true, []string{srBeta}}, + {"label", []string{"routed", "--label", "lane:a"}, false, true, []string{srAlpha, srGamma}}, + {"assignee", []string{"routed", "--assignee", "alice"}, false, true, []string{srAlpha}}, + // The store's default order is priority first, so the cut keeps P0 and P1. + {"limit", []string{"routed", "--limit", "2"}, false, true, []string{srAlpha, srBeta}}, + // The title sort is case-insensitive (workapi.SortIssues). + {"sort title", []string{"routed", "--sort", "title"}, true, true, []string{srGamma, srAlpha, srDone, srBeta}}, + {"sort priority reverse", []string{"routed", "--sort", "priority", "--reverse"}, true, true, []string{srDone, srGamma, srBeta, srAlpha}}, + {"query flag", []string{"--query", "gamma"}, false, true, []string{srGamma}}, + {"no match", []string{"nothing-matches-this"}, false, true, nil}, + {"priority-max", []string{"routed", "--priority-max", "1"}, false, false, []string{srAlpha, srBeta}}, + // --type is expanded as `bd list` expands it before the Reader validates it. + {"type alias", []string{"routed", "--type", "feat"}, false, true, []string{srGamma}}, + // Every kind of row comes back under no flag and under its own --type. An + // infra type is the one that also narrows a listing to its ephemeral rows; + // search's scope keeps the durable row of that type beside the wisp. + {"every kind of row", []string{"kinded"}, false, true, []string{skTemplate, skGate, skWisp, skPromoted}}, + {"type message keeps both planes", []string{"kinded", "--type", "message"}, false, true, []string{skWisp, skPromoted}}, + {"type gate", []string{"kinded", "--type", "gate"}, false, true, []string{skGate}}, + {"type task keeps the template", []string{"kinded", "--type", "task"}, false, true, []string{skTemplate}}, +} + +// searchRouteRefusals are flag sets every route must refuse, naming the value +// it refused, rather than answer with an empty result. +var searchRouteRefusals = []struct { + name string + args []string + value string +}{ + {"unknown type", []string{"routed", "--type", "no-such-type"}, "no-such-type"}, +} + +type searchRouteRow struct{ ID, Title, Status string } + +// searchRouteOps is how one route writes the seed. +type searchRouteOps struct { + create func(title string, args []string) string + close func(id string) + markTemplate func(id string) + promote func(id string) +} + +// seedSearchRoutes creates searchRouteSeeds through one route's writes. +func seedSearchRoutes(ops searchRouteOps) { + for _, s := range searchRouteSeeds { + id := ops.create(s.title, s.args) + switch s.then { + case srClose: + ops.close(id) + case srMarkTemplate: + ops.markTemplate(id) + case srPromote: + ops.promote(id) + } + } +} + +// runSearchRoute runs `bd search --json args...` in dir and returns its +// stdout, stderr and exit code. +func runSearchRoute(t *testing.T, bd, dir string, env []string, args ...string) (string, string, int) { + t.Helper() + cmd := exec.Command(bd, append([]string{"search", "--json"}, args...)...) + cmd.Dir = dir + cmd.Env = env + stdout, stderr, err := runCommandBuffers(t, cmd) + code := 0 + if err != nil { + ee, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("running bd search: %v", err) + } + code = ee.ExitCode() + } + return stdout.String(), stderr.String(), code +} + +func decodeSearchRoute(t *testing.T, out string) []searchRouteRow { + t.Helper() + s := strings.TrimSpace(out) + start := strings.Index(s, "[") + if start < 0 { + return nil + } + var rows []searchRouteRow + if err := json.Unmarshal([]byte(s[start:]), &rows); err != nil { + t.Fatalf("parse search JSON: %v\n%s", err, s) + } + return rows +} + +// searchRouteComparable projects rows onto what two databases with the same +// seed share (title and status, not ids); an unordered answer is a sorted set. +func searchRouteComparable(rows []searchRouteRow, ordered bool) []string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.Title+"|"+r.Status) + } + if !ordered { + slices.Sort(out) + } + return out +} + +// checkSearchRoute runs every query on one route and holds each answer to its +// want. A query listIssues cannot carry (wireCan false) must refuse on a +// remote route rather than answer without the flag, and every refusal must +// refuse on every route. It returns the decoded answers by query name, for a +// route that is compared row for row with another one over the same database. +func checkSearchRoute(t *testing.T, route, bd, dir string, env []string, remote bool) map[string][]searchRouteRow { + t.Helper() + answers := map[string][]searchRouteRow{} + for _, q := range searchRouteQueries { + stdout, stderr, code := runSearchRoute(t, bd, dir, env, q.args...) + if remote && !q.wireCan { + if code == 0 { + t.Errorf("%s: %s bd search %v succeeded; listIssues cannot carry it, so it must refuse", q.name, route, q.args) + } + continue + } + if code != 0 { + t.Errorf("%s: %s bd search %v failed (exit %d): %s", q.name, route, q.args, code, stderr) + continue + } + rows := decodeSearchRoute(t, stdout) + answers[q.name] = rows + want := slices.Clone(q.want) + if !q.ordered { + slices.Sort(want) + } + if got := searchRouteComparable(rows, q.ordered); !slices.Equal(got, want) { + t.Errorf("%s: %s bd search %v answered %v, want %v", q.name, route, q.args, got, want) + } + } + for _, r := range searchRouteRefusals { + stdout, stderr, code := runSearchRoute(t, bd, dir, env, r.args...) + if code == 0 { + t.Errorf("%s: %s bd search %v succeeded with %s; it must refuse", r.name, route, r.args, strings.TrimSpace(stdout)) + } else if !strings.Contains(stdout+stderr, r.value) { + t.Errorf("%s: %s bd search %v refused (exit %d) without naming %q:\n%s%s", r.name, route, r.args, code, r.value, stdout, stderr) + } + } + return answers +} diff --git a/cmd/bd/search_routes_proxied_test.go b/cmd/bd/search_routes_proxied_test.go new file mode 100644 index 000000000..bc5f2de88 --- /dev/null +++ b/cmd/bd/search_routes_proxied_test.go @@ -0,0 +1,66 @@ +//go:build cgo + +package main + +import ( + "os" + "os/exec" + "slices" + "testing" +) + +// TestProxiedServerSearchRoutesAgree is the proxied and remote routes of the +// `bd search` route agreement (search_routes_parity_test.go). Both answer every +// flag set as searchRouteQueries expects, the answers +// TestEmbeddedSearchRoutesAgree holds the direct route to. The remote route is +// `bd serve` over the proxied workspace (bd serve needs a SQL server, so it +// cannot serve an embedded one), so it reads the SAME database as the proxied +// route and must answer it row for row, ids and order included. +func TestProxiedServerSearchRoutesAgree(t *testing.T) { + if os.Getenv("BEADS_TEST_PROXIED_SERVER") != "1" { + t.Skip("set BEADS_TEST_PROXIED_SERVER=1 to run proxied server integration tests") + } + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "sp") + seedSearchRoutes(searchRouteOps{ + create: func(title string, args []string) string { + return bdProxiedCreate(t, bd, p.dir, append([]string{title}, args...)...).ID + }, + close: func(id string) { bdProxiedClose(t, bd, p.dir, id) }, + markTemplate: func(id string) { + res, err := openProxiedDB(t, p).Exec("UPDATE issues SET is_template = 1 WHERE id = ?", id) + if err != nil { + t.Fatalf("mark %s a template: %v", id, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("mark %s a template: %d rows affected (err %v), want 1", id, n, err) + } + }, + promote: func(id string) { + if out, err := bdProxiedRun(t, bd, p.dir, "promote", id); err != nil { + t.Fatalf("bd promote %s: %v\n%s", id, err, out) + } + }, + }) + proxied := checkSearchRoute(t, "proxied", bd, p.dir, bdProxiedEnv(p.dir), false) + + sp := startServe(t, bd, p.dir, bdProxiedEnv(p.dir)) + remoteDir := t.TempDir() + connect := exec.Command(bd, "connect", "http://"+sp.addr, "--json") + connect.Dir = remoteDir + connect.Env = bdEnv(remoteDir) + if out, err := connect.CombinedOutput(); err != nil { + t.Fatalf("bd connect: %v\n%s", err, out) + } + remote := checkSearchRoute(t, "remote", bd, remoteDir, bdEnv(remoteDir), true) + for _, q := range searchRouteQueries { + got, gotOK := remote[q.name] + want, wantOK := proxied[q.name] + if !gotOK || !wantOK { + continue // refused or failed on a route: reported above + } + if !slices.Equal(got, want) { + t.Errorf("%s: remote answered %+v, proxied answered %+v", q.name, got, want) + } + } +} diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 75f3c6668..c47e97a9a 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -81,7 +81,10 @@ var showCmd = &cobra.Command{ if len(args) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") } - currentID := resolveCurrentIssueID(ctx) + currentID, err := resolveCurrentIssueID(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } if currentID == "" { return HandleErrorRespectJSON("no current issue found (no in-progress, hooked, or recently touched issues)") } diff --git a/cmd/bd/show_current.go b/cmd/bd/show_current.go new file mode 100644 index 000000000..43e2b500a --- /dev/null +++ b/cmd/bd/show_current.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/issueops" +) + +// resolveCurrentIssueID determines the current active issue for the agent. +// Priority: in-progress assigned to actor > hooked > last touched. +// +// The lookup is the library's (issueops.FindCurrentIssue, over the Querier +// role every backend serves: queryIssues on a remote one), and both routes ask +// it: the direct route with store.Querier(), the proxied route with the +// provider's. What stays here is CLI policy: who the actor is, and the +// last-touched fallback, which is a file in this workspace. +// +// A failed read is an error. The fallback answers "nothing is in progress", +// and only reads that answered may say so: taking the http backend's refusal +// for a miss is what printed the wrong issue with exit 0. +func resolveCurrentIssueID(ctx context.Context) (string, error) { + var querier issueops.Querier + if usesProxiedServer() { + q, err := proxiedQuerier() + if err != nil { + return "", fmt.Errorf("finding the current issue: %w", err) + } + querier = q + } else if store != nil { + q, err := store.Querier() + if err != nil { + return "", fmt.Errorf("finding the current issue: %w", err) + } + querier = q + } + return resolveCurrentIssueIDFrom(ctx, querier, getActorWithGit, GetLastTouchedID) +} + +func resolveCurrentIssueIDFrom(ctx context.Context, querier issueops.Querier, currentActor func() string, fallback func() string) (string, error) { + if querier == nil { + return fallback(), nil + } + id, err := issueops.FindCurrentIssue(ctx, querier, currentActor()) + if err != nil { + return "", err + } + if id == "" { + return fallback(), nil + } + return id, nil +} diff --git a/cmd/bd/show_current_test.go b/cmd/bd/show_current_test.go index 14d0339c0..6eeec607f 100644 --- a/cmd/bd/show_current_test.go +++ b/cmd/bd/show_current_test.go @@ -3,140 +3,66 @@ package main import ( "context" "errors" - "reflect" "testing" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) -type currentIssueSearchCall struct { - query string - filter types.IssueFilter +type fixedQuerier struct { + ids []string + err error + n int } -type currentIssueSearchResponse struct { - issues []*types.Issue - err error +func (q *fixedQuerier) Query(context.Context, issueops.QueryRequest) (issueops.IssuePage, error) { + q.n++ + if q.err != nil { + return issueops.IssuePage{}, q.err + } + items := make([]*types.IssueWithCounts, 0, len(q.ids)) + for _, id := range q.ids { + items = append(items, &types.IssueWithCounts{Issue: &types.Issue{ID: id}}) + } + return issueops.IssuePage{Items: items}, nil } -type currentIssueSearchRecorder struct { - responses []currentIssueSearchResponse - calls []currentIssueSearchCall -} +// The lookup itself is issueops.FindCurrentIssue's (issueops/current_test.go). +// What the CLI owns is the fallback: last-touched answers only a lookup that +// answered and found nothing, never a failed one. -func (r *currentIssueSearchRecorder) SearchIssues(_ context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { - r.calls = append(r.calls, currentIssueSearchCall{query: query, filter: filter}) - response := r.responses[len(r.calls)-1] - return response.issues, response.err -} - -func TestResolveCurrentIssueIDFromInProgressShortCircuits(t *testing.T) { - searcher := ¤tIssueSearchRecorder{ - responses: []currentIssueSearchResponse{{ - issues: []*types.Issue{{ID: "in-progress-first"}, {ID: "in-progress-second"}}, - }}, - } - actorCalls := 0 - - got := resolveCurrentIssueIDFrom(context.Background(), searcher, func() string { - actorCalls++ - return "tester" - }, func() string { - t.Fatal("fallback called after in-progress result") - return "" - }) - - if got != "in-progress-first" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "in-progress-first") - } - if actorCalls != 1 { - t.Fatalf("actor calls = %d, want 1", actorCalls) - } - assertCurrentIssueSearchCalls(t, searcher.calls, "tester", types.StatusInProgress) -} - -func TestResolveCurrentIssueIDFromFindsHookedAfterInProgressMiss(t *testing.T) { - searcher := ¤tIssueSearchRecorder{ - responses: []currentIssueSearchResponse{ - {err: errors.New("in-progress search failed")}, - {issues: []*types.Issue{{ID: "hooked"}}}, - }, - } - - got := resolveCurrentIssueIDFrom(context.Background(), searcher, func() string { - return "tester" - }, func() string { - t.Fatal("fallback called after hooked result") - return "" - }) - - if got != "hooked" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "hooked") - } - assertCurrentIssueSearchCalls(t, searcher.calls, "tester", types.StatusInProgress, types.StatusHooked) -} - -func TestResolveCurrentIssueIDFromFallsBackAfterTwoMisses(t *testing.T) { - searcher := ¤tIssueSearchRecorder{ - responses: []currentIssueSearchResponse{{}, {}}, - } - fallbackCalls := 0 - - got := resolveCurrentIssueIDFrom(context.Background(), searcher, func() string { - return "tester" - }, func() string { - fallbackCalls++ - return "last-touched" - }) - - if got != "last-touched" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "last-touched") - } - if fallbackCalls != 1 { - t.Fatalf("fallback calls = %d, want 1", fallbackCalls) - } - assertCurrentIssueSearchCalls(t, searcher.calls, "tester", types.StatusInProgress, types.StatusHooked) -} - -func TestResolveCurrentIssueIDFromNilSearcherUsesFallback(t *testing.T) { - actorCalls := 0 - fallbackCalls := 0 - - got := resolveCurrentIssueIDFrom(context.Background(), nil, func() string { - actorCalls++ - return "tester" - }, func() string { - fallbackCalls++ - return "last-touched" - }) - - if got != "last-touched" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "last-touched") - } - if actorCalls != 0 { - t.Fatalf("actor calls = %d, want 0", actorCalls) - } - if fallbackCalls != 1 { - t.Fatalf("fallback calls = %d, want 1", fallbackCalls) +func TestResolveCurrentIssueIDFromUsesTheLookupBeforeTheFallback(t *testing.T) { + got, err := resolveCurrentIssueIDFrom(context.Background(), &fixedQuerier{ids: []string{"in-progress"}}, + func() string { return "tester" }, + func() string { t.Fatal("fallback called after a found issue"); return "" }) + if err != nil || got != "in-progress" { + t.Fatalf("got (%q, %v), want (in-progress, nil)", got, err) } } -func assertCurrentIssueSearchCalls(t *testing.T, got []currentIssueSearchCall, actor string, statuses ...types.Status) { - t.Helper() +func TestResolveCurrentIssueIDFromFallsBackOnlyOnAMiss(t *testing.T) { + got, err := resolveCurrentIssueIDFrom(context.Background(), &fixedQuerier{}, + func() string { return "tester" }, func() string { return "last-touched" }) + if err != nil || got != "last-touched" { + t.Fatalf("got (%q, %v), want (last-touched, nil)", got, err) + } +} - want := make([]currentIssueSearchCall, 0, len(statuses)) - for _, status := range statuses { - status := status - want = append(want, currentIssueSearchCall{ - query: "", - filter: types.IssueFilter{ - Status: &status, - Assignee: &actor, - }, - }) +func TestResolveCurrentIssueIDFromNeverFallsBackOnAFailure(t *testing.T) { + refusal := errors.New("not supported over HTTP") + got, err := resolveCurrentIssueIDFrom(context.Background(), &fixedQuerier{err: refusal}, + func() string { return "tester" }, + func() string { t.Fatal("fallback called after a failed read"); return "" }) + if !errors.Is(err, refusal) || got != "" { + t.Fatalf("got (%q, %v), want (\"\", an error wrapping the refusal)", got, err) } +} - if !reflect.DeepEqual(got, want) { - t.Fatalf("SearchIssues calls = %#v, want %#v", got, want) +func TestResolveCurrentIssueIDFromNilQuerierUsesFallback(t *testing.T) { + got, err := resolveCurrentIssueIDFrom(context.Background(), nil, + func() string { t.Fatal("actor resolved without a querier"); return "" }, + func() string { return "last-touched" }) + if err != nil || got != "last-touched" { + t.Fatalf("got (%q, %v), want (last-touched, nil)", got, err) } } diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 71f278c78..741bbbb63 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -100,13 +100,14 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string return err } - uw, err := proxiedOpenReadUOW(ctx) - if err != nil { + // Resolved BEFORE the unit of work opens: --current reads through the + // provider's Querier role, which runs its own read. + if err := resolveShowProxiedIDs(ctx, in); err != nil { return err } - if err := resolveShowProxiedIDs(ctx, uw, in); err != nil { - uw.Close(ctx) + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { return err } @@ -140,12 +141,15 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string // resolveShowProxiedIDs turns --current into an explicit id and rejects an // empty id list, the same checks the direct route makes before dispatching. -func resolveShowProxiedIDs(ctx context.Context, uw uow.UnitOfWork, in *showProxiedInput) error { +func resolveShowProxiedIDs(ctx context.Context, in *showProxiedInput) error { if in.currentMode { if len(in.ids) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") } - currentID := resolveCurrentIssueIDProxied(ctx, uw) + currentID, err := resolveCurrentIssueID(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } if currentID == "" { return HandleErrorRespectJSON("no current issue found (no in-progress, hooked, or recently touched issues)") } diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 65380e5a8..4183dd53b 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1101,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, and `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2396,6 +2396,8 @@ type ListIssuesParams struct { // IncludeEphemeral Include the ephemeral tier — ephemeral rows and the non-synced rows stored beside them — merged into the same order as the durable ones, whichever order `sort` named. // // It admits a TIER and takes no TYPE exclusion off, so a row whose type this operation already hides stays hidden. That includes the configured infrastructure types: ephemeral `agent`, `role` and `message` rows need `include_infra` as well as, or instead of, this one. What `include_ephemeral` alone reaches is the ephemeral rows of the types a listing already shows. + // + // A `type` this workspace calls infra is the one NARROWING it lifts. Such a type on its own reads only that type's ephemeral rows, which is where creating one puts it; with `include_ephemeral` the type's durable rows (a promoted wisp, an issue retyped after it was created) are listed as well. `include_infra` alone keeps the narrowing. IncludeEphemeral *bool `form:"include_ephemeral,omitempty" json:"include_ephemeral,omitempty"` // CreatedBefore Only issues created strictly before this instant (RFC 3339). @@ -2410,6 +2412,11 @@ type ListIssuesParams struct { // HasMetadataKey Only issues carrying this top-level metadata key. HasMetadataKey *string `form:"has_metadata_key,omitempty" json:"has_metadata_key,omitempty"` + // Q `bd search`'s free text. A row matches when its title or its id contains the text, case-insensitively; a text shaped like an issue id also matches that exact id, an id prefix, and the external reference. It narrows the listing and lifts none of its default exclusions, so a caller that wants `bd search`'s every-status, every-kind answer also sends `all`, `include_templates`, `include_gates`, `include_infra` and `include_ephemeral`. + // + // Behind the `issues.list.search` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses a search locally against a server that does not advertise the token, rather than paying for that round trip. + Q *string `form:"q,omitempty" json:"q,omitempty"` + // Sort Which of the two served total orders this page is in. `created` is `(created_at DESC, id ASC)`; `priority` is `(priority ASC, created_at DESC, id ASC)` — `bd list`'s flagless ordering, and the order `bd list --sort priority` produces. // // THE DEFAULT IS `created` AND WILL NOT CHANGE. It is what this operation served before this parameter existed, so moving it would alter which rows a truncated page contains for every client written against that behavior, with no error to notice it by. @@ -2547,7 +2554,7 @@ type CountIssuesParams struct { // Type One issue type, with `status`'s match-nothing-rather-than-fail treatment and NO shorthand alias expansion at all — unlike `GET /v0/beads/ready`'s `type`, which expands aliases. // - // It has a SECOND effect under `include_infra`: a type this workspace calls infra routes the count to the ephemeral tier. See that parameter. + // It has a SECOND effect under `include_infra`: a type this workspace calls infra routes the count to the ephemeral tier, unless `include_ephemeral` is also set. See that parameter. Type *string `form:"type,omitempty" json:"type,omitempty"` // Assignee Only issues assigned to this actor. Sending it beside `no_assignee` is not refused: the two are handed to the filter as written and answer with the empty intersection, which is `0`. @@ -2616,6 +2623,8 @@ type CountIssuesParams struct { // IncludeEphemeral Admit the EPHEMERAL PLANE — the wisps table — and nothing else. Exactly the first of `include_infra`'s four changes, with none of the other three: no type exclusion is taken off, so a row whose TYPE a default count already excludes stays excluded. // // It exists because there was no way to ask for that one thing. The write path routes on STORAGE CLASS, not type, so a `no_history` bead lives in the wisps table while remaining ordinary durable work. Reaching it needed `include_infra`, which also drops template rows of the named type — one silent undercount traded for another. + // + // Beside `include_infra` it also lifts the fourth change's routing of an infra `type`, so that type's durable rows are counted along with its ephemeral ones rather than instead of them. IncludeEphemeral *bool `form:"include_ephemeral,omitempty" json:"include_ephemeral,omitempty"` // HasMetadataKey Only issues carrying this top-level metadata key. @@ -2623,7 +2632,7 @@ type CountIssuesParams struct { // IncludeInfra Count the cardinality of `bd list --include-infra --all` instead of the durable plane. IT CHANGES FOUR THINGS AT ONCE, and they are listed rather than summarized because a caller reading "include infra" would expect one: // - // the ephemeral wisps tier is MERGED IN, picking up both wisps and the `no_history` beads that are durable work stored in that tier; template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the ephemeral tier instead of the durable one. + // the ephemeral wisps tier is MERGED IN, picking up both wisps and the `no_history` beads that are durable work stored in that tier; template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the ephemeral tier instead of the durable one — unless `include_ephemeral` is also set, which keeps the durable tier as well, as `GET /v0/beads/issues` does. // // The infra vocabulary is the WORKSPACE's, read from its configuration inside the role. A caller does not supply it and cannot — that config load is what this role exists to keep off both front doors. // diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 988f6298e..2e616f314 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -375,6 +375,9 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { MetadataFields: q.metadataFields("metadata_field"), HasMetadataKey: q.str("has_metadata_key"), + // `bd search`'s text, behind issues.list.search (CapIssuesListSearch). + Query: q.str("q"), + Limit: q.limit(), } diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index 51cbb88ab..f67dd3fe5 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -23,7 +23,10 @@ type recordingIssues struct { mu sync.Mutex ready []types.WorkFilter search []types.IssueFilter - items []*types.IssueWithCounts + // queries is the free-text argument of each SearchIssuesWithCounts call, + // index-aligned with search. + queries []string + items []*types.IssueWithCounts } func (f *recordingIssues) GetReadyWorkWithCounts(_ context.Context, filter types.WorkFilter) (domain.SearchCountsPage, error) { @@ -33,9 +36,10 @@ func (f *recordingIssues) GetReadyWorkWithCounts(_ context.Context, filter types return domain.SearchCountsPage{Items: f.items}, nil } -func (f *recordingIssues) SearchIssuesWithCounts(_ context.Context, _ string, filter types.IssueFilter) (domain.SearchCountsPage, error) { +func (f *recordingIssues) SearchIssuesWithCounts(_ context.Context, query string, filter types.IssueFilter) (domain.SearchCountsPage, error) { f.mu.Lock() f.search = append(f.search, filter) + f.queries = append(f.queries, query) f.mu.Unlock() return domain.SearchCountsPage{Items: f.items}, nil } @@ -52,6 +56,12 @@ func (f *recordingIssues) readyFilters() []types.WorkFilter { return append([]types.WorkFilter(nil), f.ready...) } +func (f *recordingIssues) searchQueries() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.queries...) +} + func (f *recordingIssues) searchFilters() []types.IssueFilter { f.mu.Lock() defer f.mu.Unlock() @@ -434,6 +444,33 @@ func TestListForwardsTheEphemeralPlaneParameter(t *testing.T) { } } +// TestListForwardsTheSearchText is the handler half of ListRequest.Query +// (issues.list.search): `q` reaches the storage seam as SearchIssues' free-text +// argument — not as TitleSearch, which matches titles only — and its absence +// sends no text at all. +func TestListForwardsTheSearchText(t *testing.T) { + for _, tc := range []struct { + name, path, want string + }{ + {"absent sends no text", "/v0/beads/issues", ""}, + {"q is the search text", "/v0/beads/issues?q=login+bug&all=true", "login bug"}, + } { + t.Run(tc.name, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + if resp := ts.get(t, tc.path); resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d, want 200", tc.path, resp.StatusCode) + } + queries := rec.searchQueries() + if len(queries) != 1 || queries[0] != tc.want { + t.Fatalf("search texts = %q, want [%q]", queries, tc.want) + } + if f := rec.searchFilters()[0]; f.TitleSearch != "" || f.TitleContains != "" { + t.Errorf("q leaked into a title-only filter: TitleSearch=%q TitleContains=%q", f.TitleSearch, f.TitleContains) + } + }) + } +} + // TestTheEphemeralPlaneParameterIsTheSkewSignal pins the version-skew half. // // A client that sends `include_ephemeral` to a server built before this diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index e4f6519cc..206a798c8 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -119,6 +119,19 @@ const CapIssuesListSort = "issues.list.sort" // count. const CapIssuesCountScope = "issues.count.scope" +// CapIssuesListSearch is the behavior capability that advertises GET +// /v0/beads/issues' `q` parameter (reads.go, spec `q` on OpListIssues): +// `bd search`'s free text, carried as issueops.ListRequest.Query. Like +// CapIssuesListSort it names a PARAMETER added to an existing operation, so it +// rides behaviorCapabilities, and it is the token the parameter's OpenAPI +// description cites for TestNewParameterOnExistingOperationHasABehaviorToken. +// +// The client (internal/httpclient) refuses a search locally against a server +// that does not advertise it, with ErrUnsupported naming the token, so a +// search is never sent to a server that would answer `unknown_parameter` and +// is never widened into an unsearched listing. +const CapIssuesListSearch = "issues.list.search" + // CapIssuesSweepWispsPlane is the behavior capability that advertises the // `wisps-plane` value of SweepRequest.tier (sweep.go, spec SweepRequest.tier): // a third, wider tier that clears the whole wisps table plus any durable bead @@ -986,7 +999,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, - CapIssuesCloseAutoCloseMolecule, + CapIssuesCloseAutoCloseMolecule, CapIssuesListSearch, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 7210f3d98..90af30763 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -638,7 +638,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", - "issues.close.autoCloseMolecule") + "issues.close.autoCloseMolecule", "issues.list.search") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 8dc0099d6..e9db266a3 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1260,6 +1260,14 @@ paths: `message` rows need `include_infra` as well as, or instead of, this one. What `include_ephemeral` alone reaches is the ephemeral rows of the types a listing already shows. + + + A `type` this workspace calls infra is the one NARROWING it lifts. + Such a type on its own reads only that type's ephemeral rows, which + is where creating one puts it; with `include_ephemeral` the type's + durable rows (a promoted wisp, an issue retyped after it was + created) are listed as well. `include_infra` alone keeps the + narrowing. schema: type: boolean default: false @@ -1291,6 +1299,25 @@ paths: description: Only issues carrying this top-level metadata key. schema: type: string + - name: q + in: query + description: >- + `bd search`'s free text. A row matches when its title or its id + contains the text, case-insensitively; a text shaped like an issue + id also matches that exact id, an id prefix, and the external + reference. It narrows the listing and lifts none of its default + exclusions, so a caller that wants `bd search`'s every-status, + every-kind answer also sends `all`, `include_templates`, + `include_gates`, `include_infra` and `include_ephemeral`. + + + Behind the `issues.list.search` behavior token: an older server + answers `unknown_parameter` for this name. A client with the + handshake's capabilities cached refuses a search locally against a + server that does not advertise the token, rather than paying for + that round trip. + schema: + type: string - name: sort in: query description: >- @@ -1871,8 +1898,8 @@ paths: It has a SECOND effect under `include_infra`: a type this workspace - calls infra routes the count to the ephemeral tier. See that - parameter. + calls infra routes the count to the ephemeral tier, unless + `include_ephemeral` is also set. See that parameter. schema: type: string - name: assignee @@ -2033,6 +2060,11 @@ paths: lives in the wisps table while remaining ordinary durable work. Reaching it needed `include_infra`, which also drops template rows of the named type — one silent undercount traded for another. + + + Beside `include_infra` it also lifts the fourth change's routing of + an infra `type`, so that type's durable rows are counted along with + its ephemeral ones rather than instead of them. schema: type: boolean default: false @@ -2055,7 +2087,9 @@ paths: template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the - ephemeral tier instead of the durable one. + ephemeral tier instead of the durable one — unless + `include_ephemeral` is also set, which keeps the durable tier as + well, as `GET /v0/beads/issues` does. The infra vocabulary is the WORKSPACE's, read from its configuration @@ -8195,7 +8229,7 @@ components: `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than - silently answering `unknown_parameter`, and + silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than @@ -8213,7 +8247,10 @@ components: `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the - member unknown. One behavior token is + member unknown, and `issues.list.search`, which announces that + `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see + that parameter), rather than silently answering + `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 6604685ac..29d527ff6 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -5855,6 +5855,14 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:q", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:sort", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 265a370e8..2dbd3ee07 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "batchcloser.go", "batchcreator.go", "batchgetter.go", + "blocked_recomputer.go", "bridge.go", "ca.go", "ca_permissions_other.go", @@ -116,6 +117,7 @@ go_test( "insecure_credential_guard_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", + "list_search_test.go", "list_sort_test.go", "metadatacas_test.go", "molecule_test.go", diff --git a/internal/httpclient/blocked_recomputer.go b/internal/httpclient/blocked_recomputer.go new file mode 100644 index 000000000..603dcf8ee --- /dev/null +++ b/internal/httpclient/blocked_recomputer.go @@ -0,0 +1,12 @@ +package httpclient + +import "github.com/steveyegge/beads/internal/storage" + +// BlockedStateMaintainedByServer satisfies storage.ServerMaintainedBlockedState: +// the server derives and maintains is_blocked (issueops.BlockedStateInvariant's +// local-write clause runs on every write the server commits) and the column +// lives in the server's database, so storage.RecomputeBlocked answers +// storage.BlockedMaintainedByServer for this store and dials nothing. +func (s *Store) BlockedStateMaintainedByServer() bool { return true } + +var _ storage.ServerMaintainedBlockedState = (*Store)(nil) diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 41be7e31e..3db3f19bd 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -295,6 +295,7 @@ func ListParams(req issueops.ListRequest) (url.Values, error) { b.list("exclude_label", req.ExcludeLabels) b.str("parent", req.ParentID) + b.str("q", req.Query) b.boolean("all", req.AllFlag) b.boolean("include_templates", req.IncludeTemplates) diff --git a/internal/httpclient/encode/parentwalk.go b/internal/httpclient/encode/parentwalk.go index aa75cb551..f9677388e 100644 --- a/internal/httpclient/encode/parentwalk.go +++ b/internal/httpclient/encode/parentwalk.go @@ -212,12 +212,21 @@ func recognizeIntents(f types.IssueFilter, cfg ListConfig) (walkIntents, error) } // The same rule for the plane, and it has TWO suppressors rather than one: // `--include-infra` admits the wisp plane as well as the infra types, and an - // infra `--type` routes to the plane on its own. Either one leaves SkipWisps - // false whatever the ephemeral flag was, so under either of them the pair is - // the same question as the infra intent alone — `--all --no-pinned` and - // `--status all` again — and inventing the second parameter would be a - // distinction the re-derivation cannot see. - if intents.IncludeInfra || cfg.IsInfra(issueType) { + // infra `--type` admits it on its own. Either one leaves SkipWisps false + // whatever the ephemeral flag was, so SkipWisps cannot carry the intent + // under either of them. + // + // Under `--include-infra` alone the pair is the same question as the infra + // intent — `--all --no-pinned` and `--status all` again — and inventing the + // second parameter would be a distinction the re-derivation cannot see. An + // infra `--type` is different: it also narrows the read to that type's + // ephemeral rows, and the ephemeral flag LIFTS that narrowing (so durable + // rows of the type come back too), so there the plane intent leaves its + // trace in Ephemeral instead — present without the flag, absent with it. + switch { + case cfg.IsInfra(issueType): + intents.IncludeEphemeral = f.Ephemeral == nil + case intents.IncludeInfra: intents.IncludeEphemeral = false } diff --git a/internal/httpclient/encode/parentwalk_test.go b/internal/httpclient/encode/parentwalk_test.go index f94ac24b1..f51b81a12 100644 --- a/internal/httpclient/encode/parentwalk_test.go +++ b/internal/httpclient/encode/parentwalk_test.go @@ -127,6 +127,30 @@ func TestTheParentWalkInvertsTheDerivedDefaults(t *testing.T) { in: textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true}), want: url.Values{"parent": {"bd-root"}, "include_infra": {"true"}}, }, + { + // An infra --type admits the plane on its own too, AND narrows the + // read to that type's ephemeral rows, so a bare one is no plane + // intent at all: the server re-derives the narrowing from the type. + name: "an infra --type alone does not invent an ephemeral intent", + in: textListing(issueops.ListRequest{IssueType: "message"}), + want: url.Values{"parent": {"bd-root"}, "type": {"message"}}, + }, + { + // With an infra --type the ephemeral flag is NOT redundant: it lifts + // the type's ephemeral-only narrowing, so the durable rows of that + // type (a promoted wisp, a retyped issue) come back too. Dropping the + // parameter here would ask the server the narrower question. + name: "--include-ephemeral with an infra --type keeps the plane intent", + in: textListing(issueops.ListRequest{IncludeEphemeral: true, IssueType: "message"}), + want: url.Values{"parent": {"bd-root"}, "type": {"message"}, "include_ephemeral": {"true"}}, + }, + { + // --include-infra is redundant beside an infra --type whatever the + // ephemeral flag says, so the pair still reads as the flag alone. + name: "--include-ephemeral --include-infra with an infra --type is the same question as --include-ephemeral", + in: textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true, IssueType: "message"}), + want: url.Values{"parent": {"bd-root"}, "type": {"message"}, "include_ephemeral": {"true"}}, + }, { name: "an explicit status travels verbatim under its own parameter", in: textListing(issueops.ListRequest{Status: "in_progress"}), @@ -446,6 +470,8 @@ func TestTheParentWalkRoundTripsThroughTheServerDecoder(t *testing.T) { {"--include-ephemeral alone", textListing(issueops.ListRequest{IncludeEphemeral: true})}, {"--include-ephemeral --include-infra", textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true})}, {"--include-ephemeral with an infra --type", textListing(issueops.ListRequest{IncludeEphemeral: true, IssueType: "agent"})}, + {"--include-ephemeral --include-infra with an infra --type", textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true, IssueType: "agent"})}, + {"--include-infra with an infra --type", textListing(issueops.ListRequest{IncludeInfra: true, IssueType: "agent"})}, {"--include-templates alone", textListing(issueops.ListRequest{IncludeTemplates: true})}, {"an explicit status", textListing(issueops.ListRequest{Status: "in_progress"})}, {"a multi-status filter", textListing(issueops.ListRequest{Status: "open,in_progress"})}, diff --git a/internal/httpclient/encode/roundtrip_test.go b/internal/httpclient/encode/roundtrip_test.go index 6d09ba6d7..6e2971693 100644 --- a/internal/httpclient/encode/roundtrip_test.go +++ b/internal/httpclient/encode/roundtrip_test.go @@ -182,6 +182,7 @@ func roundTripCases() []roundTripCase { ExcludeLabels: []string{"delta", "epsilon"}, ParentID: "bd-epic1", + Query: "login bug", AllFlag: true, IncludeTemplates: true, diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index 9e6523a51..462598658 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -329,6 +329,10 @@ func listTable() Table { refused("TitleSearch", "E-ListRequest.TitleSearch"), refused("SpecPrefix", "E-ListRequest.SpecPrefix"), refused("IDFilter", "E-ListRequest.IDFilter"), + // Behind issues.list.search: httpReader.List refuses a request + // carrying it, before dialing, unless the handshake advertises the + // token (role_reader.go refuseUnservedSearch). + param("Query", "q"), param("Labels", "label"), param("LabelsAny", "label_any"), param("ExcludeLabels", "exclude_label"), diff --git a/internal/httpclient/list_search_test.go b/internal/httpclient/list_search_test.go new file mode 100644 index 000000000..dfce7885c --- /dev/null +++ b/internal/httpclient/list_search_test.go @@ -0,0 +1,67 @@ +package httpclient + +import ( + "errors" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +func listRoleWithSnapshot(t *testing.T, w *countingWire, snap *apigen.ContextResponse) issueops.Reader { + t.Helper() + reader, err := New(testTarget(t), w, snap).IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return reader +} + +// TestListQueryRidesQBehindTheSearchCapability is S26's client half: a +// ListRequest carrying Query reaches listIssues as `q` once the handshake +// advertises issues.list.search. +func TestListQueryRidesQBehindTheSearchCapability(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListSearch}} + w := newCountingWire(`{"items":[],"has_more":false}`) + limit := 10 + if _, err := listRoleWithSnapshot(t, w, served).List(t.Context(), issueops.ListRequest{Query: "login bug", Limit: &limit}); err != nil { + t.Fatalf("List: %v", err) + } + if got := w.query(t)["q"]; !slices.Equal(got, []string{"login bug"}) { + t.Errorf("query[q] = %v, want [login bug] (full query: %v)", got, w.query(t)) + } +} + +// TestListQueryRefusesLocallyWhenTheServerLacksTheSearchCapability is the skew +// side: against a server that does not advertise issues.list.search (the +// deployed reference server, for one), a search refuses BEFORE dialing with +// the typed capability error. Sending it would cost a guaranteed 400 +// unknown_parameter; dropping `q` would answer the unsearched listing as the +// search. +func TestListQueryRefusesLocallyWhenTheServerLacksTheSearchCapability(t *testing.T) { + masked := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListSort}} + w := newCountingWire(`{"items":[],"has_more":false}`) + _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), issueops.ListRequest{Query: "login bug"}) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("List error = %v, want *storage.ErrUnsupported", err) + } + if unsup.Capability != wire.CapListSearch || unsup.Op != "Reader.List" { + t.Errorf("refusal = {Op: %q, Capability: %q}, want {Reader.List, %s}", unsup.Op, unsup.Capability, wire.CapListSearch) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.dialed)) + } + + // A listing without Query is unaffected by the missing token. + limit := 10 + if _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), issueops.ListRequest{Limit: &limit}); err != nil { + t.Fatalf("List without Query against the same server: %v", err) + } + if q := w.query(t); q.Has("q") { + t.Errorf("a listing without Query sent q: %v", q) + } +} diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index d1758a083..77eef9a2b 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -5,6 +5,7 @@ package httpclient import ( "context" "net/http" + "slices" "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/encode" @@ -62,6 +63,9 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue // List serves listIssues, through the pager in list_walk.go. func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + if err := r.refuseUnservedSearch(ctx, req); err != nil { + return issueops.IssuePage{}, err + } params, err := encode.ListParams(req) if err != nil { return issueops.IssuePage{}, r.store.inexpressible("Reader.List", err) @@ -69,6 +73,30 @@ func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueop return r.store.walkIssues(ctx, params, req) } +// refuseUnservedSearch is the pre-dial gate for ListRequest.Query, which rides +// listIssues' `q` only on a server advertising issues.list.search. An older +// server answers `q` with 400 unknown_parameter, so a search against it is +// refused HERE, with the typed capability error naming the token: never a +// round trip that is refused anyway, and never the unsearched listing passed +// off as the search. +// +// listIssues is a baseline operation, so dispatch preflights nothing for it; +// this is the one handshake a search costs, and a request without Query never +// asks for it. +func (r httpReader) refuseUnservedSearch(ctx context.Context, req issueops.ListRequest) error { + if req.Query == "" { + return nil + } + snap, err := r.store.snapshot(ctx) + if err != nil { + return err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapListSearch) { + return nil + } + return r.store.unsupportedCapability("Reader.List", wire.CapListSearch) +} + // Get serves getIssue, including the two landed include parameters that retired // ledger row L4. // diff --git a/internal/httpclient/served_reader_test.go b/internal/httpclient/served_reader_test.go index 696589a99..be604e36d 100644 --- a/internal/httpclient/served_reader_test.go +++ b/internal/httpclient/served_reader_test.go @@ -54,6 +54,7 @@ func servedReaderFixture(t *testing.T, name string) conformance.ReaderFixture { CreateWisp: c.seedIssue, AddDependency: c.seedDependency, AddComment: c.seedComment, + PromoteWisp: c.seedPromotion, } } @@ -181,6 +182,13 @@ func TestServedReaderListEmptyPageIsWellFormed(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, t.Context(), servedReaderFixture(t, "rdr")) } +// TestServedReaderListQueryMatchesTheTitleOrTheID runs the search-text case as +// written: it scopes itself with a label, and Query rides listIssues' `q` +// parameter behind issues.list.search, which this in-process server serves. +func TestServedReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, t.Context(), servedReaderFixture(t, "rdr")) +} + func TestServedReaderListMaxRowsIsHonored(t *testing.T) { skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, "the case scopes itself with IDFilter; the cap it pins is honored over this wire and exercised by "+ diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index 23b16b02e..b94c31275 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -312,6 +312,10 @@ func (c *servedComposition) seedComment(ctx context.Context, issueID, author, te return err } +func (c *servedComposition) seedPromotion(ctx context.Context, id, actor string) error { + return c.reference.PromoteFromEphemeral(ctx, id, actor) +} + // queryScalar is the composition's raw-row hook: a single-row query against the // REFERENCE store, which is where the rows a contract seeded actually live. // diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index fa5990c7a..2b8fc11bd 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -154,6 +154,15 @@ const CapListSort = "issues.list.sort" // parity gate compares this client's whole vocabulary with the server's. const CapCountScope = "issues.count.scope" +// CapListSearch is the behavior capability announcing that listIssues honors +// `q`, `bd search`'s free text (issueops.ListRequest.Query), spelled exactly +// as httpapi's CapIssuesListSearch. The Reader's List reads it off the +// handshake snapshot and refuses a request that carries Query LOCALLY, with a +// typed capability error naming this token, when the server does not +// advertise it — so a search never dials a server that would answer +// `unknown_parameter`, and is never widened into the unsearched listing. +const CapListSearch = "issues.list.search" + // CapBatchApplyLarge is the behavior capability announcing that issues.batchApply // accepts a batch larger than the compiled-in floor (100 items), up to the // raised ceiling (1000), spelled exactly as httpapi's constant of the same name @@ -382,7 +391,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, - CapCloseAutoCloseMolecule, + CapCloseAutoCloseMolecule, CapListSearch, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index ea4d4d21d..966354325 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -148,6 +148,9 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapListSort != httpapi.CapIssuesListSort { t.Errorf("CapListSort = %q, server says %q", CapListSort, httpapi.CapIssuesListSort) } + if CapListSearch != httpapi.CapIssuesListSearch { + t.Errorf("CapListSearch = %q, server says %q", CapListSearch, httpapi.CapIssuesListSearch) + } if CapCountScope != httpapi.CapIssuesCountScope { t.Errorf("CapCountScope = %q, server says %q", CapCountScope, httpapi.CapIssuesCountScope) } @@ -201,11 +204,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, - httpapi.CapIssuesCloseAutoCloseMolecule, + httpapi.CapIssuesCloseAutoCloseMolecule, httpapi.CapIssuesListSearch, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ - "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority and issues.close.autoCloseMolecule tokens", behaviorCapabilities) + "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit, "+ + "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule and issues.list.search tokens", behaviorCapabilities) } } diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index c3390f6ac..acb3b8e08 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "batch.go", "batch_apply.go", "batch_close_policy.go", + "blocked_recomputer.go", "bulk_issues.go", "child_counter_reservation.go", "compaction.go", @@ -81,6 +82,7 @@ go_test( srcs = [ "batch_apply_test.go", "batch_close_policy_test.go", + "blocked_recomputer_test.go", "contract_leg_registry_test.go", "directory_size_test.go", "hook_batch_applier_test.go", diff --git a/internal/storage/blocked_recomputer.go b/internal/storage/blocked_recomputer.go new file mode 100644 index 000000000..62d12b7ee --- /dev/null +++ b/internal/storage/blocked_recomputer.go @@ -0,0 +1,70 @@ +package storage + +import ( + "context" +) + +// BlockedMaintainedByServer is RecomputeBlockedResult.MaintainedBy for a +// backend whose blocked state lives in, and is maintained by, a remote server. +const BlockedMaintainedByServer = "server" + +// RecomputeBlockedResult is what one is_blocked repair did. +type RecomputeBlockedResult struct { + // RowsCorrected is how many rows' persisted flag changed. A consistent + // store corrects none. + RowsCorrected int + // MaintainedBy is "" when THIS backend recomputed its own column, and + // BlockedMaintainedByServer when the column belongs to a remote server + // that maintains it, in which case nothing was recomputed here and + // RowsCorrected is 0. A caller that must tell "nothing was stale" from + // "nothing was checked here" reads it. + MaintainedBy string +} + +// ServerMaintainedBlockedState is implemented by a store whose blocked state a +// remote server derives and keeps — today the http backend. +type ServerMaintainedBlockedState interface { + BlockedStateMaintainedByServer() bool +} + +// RecomputeBlocked is the one library entry for the is_blocked repair +// (`bd recompute-blocked`), the named repair of +// issueops.BlockedStateInvariant's merge clause — the one way that +// invariant's persisted column can be stale. Every store goes through it. The +// `--proxied-server` route deliberately does not: it runs +// versioncontrolops.RecomputeAllBlockedOnConn (the guard, recompute, staging +// set and commit message a local store uses) on a pinned non-transactional +// connection, because the repair's DOLT_ADD/DOLT_COMMIT are stored procedures +// Dolt refuses inside an explicit transaction and its commit must stage +// `issues` alone (cmd/bd/recompute_blocked_proxied_server.go). +// +// A LOCAL store (BlockedRecomputer) recomputes its whole column from the +// dependency graph and commits the result, idempotently. +// +// A REMOTE store (ServerMaintainedBlockedState) recomputes nothing and answers +// BlockedMaintainedByServer. Every write its client makes is committed by the +// server's own store under the invariant's local-write clause, so the client +// cannot leave the column stale and holds no copy of it; the merge clause's +// staleness happens to the server's database, and its repair is this same +// entry run against that database, in the server's workspace. There is no +// wire operation for it: a whole-column rewrite any bearer of a workspace +// credential could trigger, for state no client can have caused, is not one +// to publish. +// +// It looks through the decorator chain, as the repair fires no hooks. Any +// other store refuses with *ErrUnsupported. +func RecomputeBlocked(ctx context.Context, s DoltStorage) (RecomputeBlockedResult, error) { + inner := UnwrapStore(s) + if remote, ok := inner.(ServerMaintainedBlockedState); ok && remote.BlockedStateMaintainedByServer() { + return RecomputeBlockedResult{MaintainedBy: BlockedMaintainedByServer}, nil + } + local, ok := inner.(BlockedRecomputer) + if !ok { + return RecomputeBlockedResult{}, &ErrUnsupported{Op: "RecomputeAllBlocked", Backend: "this storage"} + } + changed, err := local.RecomputeAllBlocked(ctx) + if err != nil { + return RecomputeBlockedResult{}, err + } + return RecomputeBlockedResult{RowsCorrected: changed}, nil +} diff --git a/internal/storage/blocked_recomputer_test.go b/internal/storage/blocked_recomputer_test.go new file mode 100644 index 000000000..6ece323c8 --- /dev/null +++ b/internal/storage/blocked_recomputer_test.go @@ -0,0 +1,61 @@ +package storage + +import ( + "context" + "errors" + "testing" +) + +type localRecomputeStore struct { + DoltStorage + changed int + err error + calls int +} + +func (s *localRecomputeStore) RecomputeAllBlocked(context.Context) (int, error) { + s.calls++ + return s.changed, s.err +} + +type serverMaintainedStore struct { + localRecomputeStore +} + +func (*serverMaintainedStore) BlockedStateMaintainedByServer() bool { return true } + +type plainStore struct{ DoltStorage } + +func TestRecomputeBlockedRunsTheLocalRepair(t *testing.T) { + s := &localRecomputeStore{changed: 3} + got, err := RecomputeBlocked(context.Background(), s) + if err != nil || got != (RecomputeBlockedResult{RowsCorrected: 3}) || s.calls != 1 { + t.Fatalf("RecomputeBlocked = (%+v, %v) after %d repairs, want ({3 \"\"}, nil) after 1", got, err, s.calls) + } + s.err = errors.New("dirty graph") + if _, err := RecomputeBlocked(context.Background(), s); !errors.Is(err, s.err) { + t.Fatalf("RecomputeBlocked error = %v, want the repair's own error", err) + } +} + +// TestRecomputeBlockedAnswersServerMaintainedWithoutRepairing pins S25: a store +// whose server maintains the column recomputes nothing, even if it ALSO offers +// a local repair, and says who maintains it. +func TestRecomputeBlockedAnswersServerMaintainedWithoutRepairing(t *testing.T) { + s := &serverMaintainedStore{localRecomputeStore{changed: 7}} + got, err := RecomputeBlocked(context.Background(), s) + if err != nil || got != (RecomputeBlockedResult{MaintainedBy: BlockedMaintainedByServer}) { + t.Fatalf("RecomputeBlocked = (%+v, %v), want ({0 server}, nil)", got, err) + } + if s.calls != 0 { + t.Fatalf("the local repair ran %d times for a server-maintained store, want 0", s.calls) + } +} + +func TestRecomputeBlockedRefusesAStoreWithNeither(t *testing.T) { + _, err := RecomputeBlocked(context.Background(), plainStore{}) + var unsup *ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("RecomputeBlocked error = %v, want *ErrUnsupported", err) + } +} diff --git a/internal/storage/dolt/reader_contract_test.go b/internal/storage/dolt/reader_contract_test.go index 2bc7a4124..99a1ff884 100644 --- a/internal/storage/dolt/reader_contract_test.go +++ b/internal/storage/dolt/reader_contract_test.go @@ -134,6 +134,12 @@ func TestReaderListEmptyPageIsWellFormed(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, ctx, fixture) } +func TestReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { + fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") + defer cleanup() + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, ctx, fixture) +} + func TestReaderGetResolvesTheExactIDAcrossBothPlanes(t *testing.T) { fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") defer cleanup() @@ -297,6 +303,7 @@ func newDoltReaderFixture(t *testing.T, prefix string) (conformance.ReaderFixtur _, err := store.AddIssueComment(ctx, issueID, author, text) return err }, + PromoteWisp: store.PromoteFromEphemeral, } return fixture, ctx, func() { cancel() diff --git a/internal/storage/embeddeddolt/reader_contract_test.go b/internal/storage/embeddeddolt/reader_contract_test.go index 7d792ad2c..2bbdfbc9f 100644 --- a/internal/storage/embeddeddolt/reader_contract_test.go +++ b/internal/storage/embeddeddolt/reader_contract_test.go @@ -133,6 +133,12 @@ func TestEmbeddedReaderListEmptyPageIsWellFormed(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, ctx, newEmbeddedReaderFixture(t, "rdr")) } +func TestEmbeddedReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, ctx, newEmbeddedReaderFixture(t, "rdr")) +} + func TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() @@ -276,5 +282,6 @@ func newEmbeddedReaderFixture(t *testing.T, prefix string) conformance.ReaderFix _, err := te.store.AddIssueComment(ctx, issueID, author, text) return err }, + PromoteWisp: te.store.PromoteFromEphemeral, } } diff --git a/internal/storage/uow/issue_reader.go b/internal/storage/uow/issue_reader.go index 5f9bcc147..8c62b8779 100644 --- a/internal/storage/uow/issue_reader.go +++ b/internal/storage/uow/issue_reader.go @@ -122,7 +122,7 @@ func (r *issueReader) List(ctx context.Context, req publicops.ListRequest) (publ if req.ReadyFlag { page, err = uw.IssueUseCase().GetReadyWorkWithCounts(ctx, workapi.ReadyFilterFromIssueFilter(filter)) } else { - page, err = uw.IssueUseCase().SearchIssuesWithCounts(ctx, "", filter) + page, err = uw.IssueUseCase().SearchIssuesWithCounts(ctx, req.Query, filter) } if err != nil { return publicops.IssuePage{}, err diff --git a/internal/storage/uow/reader_contract_test.go b/internal/storage/uow/reader_contract_test.go index 427219c4e..cd38f36c4 100644 --- a/internal/storage/uow/reader_contract_test.go +++ b/internal/storage/uow/reader_contract_test.go @@ -92,6 +92,9 @@ func TestReaderContract(t *testing.T) { t.Run("ListEmptyPageIsWellFormed", func(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, ctx, fixture) }) + t.Run("ListQueryMatchesTheTitleOrTheID", func(t *testing.T) { + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, ctx, fixture) + }) t.Run("GetResolvesTheExactIDAcrossBothPlanes", func(t *testing.T) { conformance.RunReaderGetResolvesTheExactIDAcrossBothPlanes(t, ctx, fixture) }) @@ -207,5 +210,10 @@ func newUOWReaderFixture(t *testing.T, provider UnitOfWorkProvider, prefix strin return "bd: comment " + issueID, err }) }, + PromoteWisp: func(ctx context.Context, id, actor string) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "bd: promote " + id, uw.IssueUseCase().PromoteWisp(ctx, id, actor) + }) + }, } } diff --git a/internal/types/types.go b/internal/types/types.go index dffc85811..7a0edf045 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1230,7 +1230,8 @@ type IssueWithCounts struct { // WHICH SURFACES SET IT, because this type is shared by more than the // listing and an absent marker means different things on them. It is set // by the page epilogue behind issueops.Reader.List — `bd list --json` on - // both routes, its --ready arm included, and GET /v0/beads/issues. It is + // both routes, its --ready arm included, `bd search --json` on every + // route, and GET /v0/beads/issues. It is // NOT set by Reader.Ready (GET /v0/beads/ready) or by the claim response // that returns this type, so on those an absent marker says nothing about // whether a row's comments exist: read comment_count there. Extending the diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 66c1f509b..0931b468b 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -160,7 +160,7 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt } if in.IncludeInfra { - applyCountIncludeInfra(&filter, in.IssueType, cfg) + applyCountIncludeInfra(&filter, in.IssueType, in.IncludeEphemeral, cfg) } else if !in.IncludeEphemeral { filter.SkipWisps = true } @@ -178,12 +178,15 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt // - template molecules are excluded (list's default); // - gate beads are excluded unless requested via --type gate; // - counting an infra type (agent/role/message, or the store-configured set) -// routes to the ephemeral wisps tier, like list's infra-type listing. +// routes to the ephemeral wisps tier, like list's infra-type listing — +// unless includeEphemeral is also set, which lifts that narrowing on both +// builders, so the durable rows of that type are counted as they are +// listed. // // A count without IncludeInfra never calls this. It keeps its historical // durable-only semantics unless IncludeEphemeral admits the wisps tier: the // first of the changes above, with none of the rest. -func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, cfg ListConfig) { +func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, includeEphemeral bool, cfg ListConfig) { filter.SkipWisps = false isTemplate := false @@ -193,7 +196,7 @@ func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, cfg Lis filter.ExcludeTypes = append(filter.ExcludeTypes, "gate") } - if issueType != "" && cfg.IsInfra(issueType) { + if issueType != "" && cfg.IsInfra(issueType) && !includeEphemeral { ephemeral := true filter.Ephemeral = &ephemeral } diff --git a/internal/workapi/count_test.go b/internal/workapi/count_test.go index 2d8b102fe..45e7e603a 100644 --- a/internal/workapi/count_test.go +++ b/internal/workapi/count_test.go @@ -16,57 +16,76 @@ import ( // dimensions are the wisps merge (SkipWisps), template exclusion (IsTemplate), // the default gate exclusion (ExcludeTypes), and infra-type routing to the // ephemeral tier (Ephemeral). +// +// Each type runs twice, the second time with --include-ephemeral on both +// sides. That flag lifts the infra-type routing on the list side (an +// infra-typed row can live in the durable table too, and the flag promises +// the durable answer plus the plane), so a count that kept routing would read +// only the wisp tier while the listing it mirrors reads both: the `message` +// arm is the one that fails if the two builders stop moving in lockstep. func TestBuildCountFilterIncludeInfraMirrorsListFilter(t *testing.T) { cfg := ListConfig{} - for _, issueType := range []string{"", "task", "gate", "message"} { - name := issueType - if name == "" { - name = "none" + for _, includeEphemeral := range []bool{false, true} { + for _, issueType := range []string{"", "task", "gate", "message"} { + name := issueType + if name == "" { + name = "none" + } + if includeEphemeral { + name += "_include_ephemeral" + } + t.Run("type_"+name, func(t *testing.T) { + assertCountMirrorsListIncludeInfra(t, cfg, issueType, includeEphemeral) + }) } - t.Run("type_"+name, func(t *testing.T) { - want, err := BuildListFilter(issueops.ListRequest{ - AllFlag: true, IncludeInfra: true, IssueType: issueType, - }, cfg) - if err != nil { - t.Fatalf("BuildListFilter(%q): %v", issueType, err) - } - got, err := BuildCountFilter(issueops.CountRequest{ - IncludeInfra: true, IssueType: issueType, - }, cfg) - if err != nil { - t.Fatalf("BuildCountFilter(%q): %v", issueType, err) - } + } +} - if got.SkipWisps != want.SkipWisps { - t.Errorf("SkipWisps = %v, list --include-infra --all uses %v", got.SkipWisps, want.SkipWisps) - } - if !reflect.DeepEqual(got.IsTemplate, want.IsTemplate) { - t.Errorf("IsTemplate = %v, list --include-infra --all uses %v", countPtrStr(got.IsTemplate), countPtrStr(want.IsTemplate)) - } - if !reflect.DeepEqual(got.ExcludeTypes, want.ExcludeTypes) { - t.Errorf("ExcludeTypes = %v, list --include-infra --all uses %v", got.ExcludeTypes, want.ExcludeTypes) - } - if !reflect.DeepEqual(got.Ephemeral, want.Ephemeral) { - t.Errorf("Ephemeral = %v, list --include-infra --all uses %v", countPtrStr(got.Ephemeral), countPtrStr(want.Ephemeral)) - } - if !reflect.DeepEqual(got.IssueType, want.IssueType) { - t.Errorf("IssueType = %v, list --include-infra --all uses %v", got.IssueType, want.IssueType) - } - // A count defaults to all statuses and all pinned states, which is - // exactly what list's --all flag selects. - if !reflect.DeepEqual(got.Status, want.Status) { - t.Errorf("Status = %v, list --include-infra --all uses %v", got.Status, want.Status) - } - if !reflect.DeepEqual(got.Statuses, want.Statuses) { - t.Errorf("Statuses = %v, list --include-infra --all uses %v", got.Statuses, want.Statuses) - } - if !reflect.DeepEqual(got.ExcludeStatus, want.ExcludeStatus) { - t.Errorf("ExcludeStatus = %v, list --include-infra --all uses %v", got.ExcludeStatus, want.ExcludeStatus) - } - if !reflect.DeepEqual(got.Pinned, want.Pinned) { - t.Errorf("Pinned = %v, list --include-infra --all uses %v", countPtrStr(got.Pinned), countPtrStr(want.Pinned)) - } - }) +// assertCountMirrorsListIncludeInfra compares one `bd count --include-infra` +// filter against the `bd list --include-infra --all` filter it mirrors. +func assertCountMirrorsListIncludeInfra(t *testing.T, cfg ListConfig, issueType string, includeEphemeral bool) { + t.Helper() + want, err := BuildListFilter(issueops.ListRequest{ + AllFlag: true, IncludeInfra: true, IssueType: issueType, IncludeEphemeral: includeEphemeral, + }, cfg) + if err != nil { + t.Fatalf("BuildListFilter(%q): %v", issueType, err) + } + got, err := BuildCountFilter(issueops.CountRequest{ + IncludeInfra: true, IssueType: issueType, IncludeEphemeral: includeEphemeral, + }, cfg) + if err != nil { + t.Fatalf("BuildCountFilter(%q): %v", issueType, err) + } + + if got.SkipWisps != want.SkipWisps { + t.Errorf("SkipWisps = %v, list --include-infra --all uses %v", got.SkipWisps, want.SkipWisps) + } + if !reflect.DeepEqual(got.IsTemplate, want.IsTemplate) { + t.Errorf("IsTemplate = %v, list --include-infra --all uses %v", countPtrStr(got.IsTemplate), countPtrStr(want.IsTemplate)) + } + if !reflect.DeepEqual(got.ExcludeTypes, want.ExcludeTypes) { + t.Errorf("ExcludeTypes = %v, list --include-infra --all uses %v", got.ExcludeTypes, want.ExcludeTypes) + } + if !reflect.DeepEqual(got.Ephemeral, want.Ephemeral) { + t.Errorf("Ephemeral = %v, list --include-infra --all uses %v", countPtrStr(got.Ephemeral), countPtrStr(want.Ephemeral)) + } + if !reflect.DeepEqual(got.IssueType, want.IssueType) { + t.Errorf("IssueType = %v, list --include-infra --all uses %v", got.IssueType, want.IssueType) + } + // A count defaults to all statuses and all pinned states, which is + // exactly what list's --all flag selects. + if !reflect.DeepEqual(got.Status, want.Status) { + t.Errorf("Status = %v, list --include-infra --all uses %v", got.Status, want.Status) + } + if !reflect.DeepEqual(got.Statuses, want.Statuses) { + t.Errorf("Statuses = %v, list --include-infra --all uses %v", got.Statuses, want.Statuses) + } + if !reflect.DeepEqual(got.ExcludeStatus, want.ExcludeStatus) { + t.Errorf("ExcludeStatus = %v, list --include-infra --all uses %v", got.ExcludeStatus, want.ExcludeStatus) + } + if !reflect.DeepEqual(got.Pinned, want.Pinned) { + t.Errorf("Pinned = %v, list --include-infra --all uses %v", countPtrStr(got.Pinned), countPtrStr(want.Pinned)) } } diff --git a/internal/workapi/list.go b/internal/workapi/list.go index 1798824c1..e5f9d7864 100644 --- a/internal/workapi/list.go +++ b/internal/workapi/list.go @@ -367,11 +367,6 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter } } - if cfg.IsInfra(in.IssueType) { - ephemeral := true - filter.Ephemeral = &ephemeral - } - if in.ParentID != "" { pid := in.ParentID filter.ParentID = &pid @@ -415,7 +410,8 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter // account of WHAT IT IS, so that a "show everything" frontend has exactly one // thing to skip. Two kinds live here: the type-level exclusions that make the // default listing show durable work only (templates, gates, infra types), and -// the plane decision — whether the wisps TABLE is read at all. +// the plane decision — whether the wisps TABLE is read at all, and for a named +// infra type whether only its ephemeral rows are. // // IncludeAllTypes lifts them ALL by skipping this function entirely, which is // what makes it future-proof: a suppression added here is automatically lifted @@ -442,14 +438,28 @@ func applyTypeSuppressions(in issueops.ListRequest, cfg ListConfig, filter *type // The plane bit. Three requests admit the wisp table: an explicit // IncludeEphemeral, IncludeInfra (which admits the plane AND drops the - // infra-type exclusions above), and naming an infra type (which routed to - // the plane alone in BuildListFilter). Everything else is the durable - // listing. IncludeAllTypes is a fourth, via the early return above — it is - // the union of these flags, so it must decide the plane too, not only the - // type exclusions. + // infra-type exclusions above), and naming an infra type (which narrows to + // its ephemeral rows, below). Everything else is the durable listing. + // IncludeAllTypes is a fourth, via the early return above — it is the union + // of these flags, so it must decide the plane too, not only the type + // exclusions. if !in.IncludeEphemeral && !in.IncludeInfra && (in.IssueType == "" || !cfg.IsInfra(in.IssueType)) { filter.SkipWisps = true } + + // Naming an infra type narrows the read to its EPHEMERAL rows alone, which + // is where creating one puts it. IncludeEphemeral lifts that narrowing, + // because it promises the durable answer PLUS the plane, never the plane + // instead: an infra-typed row can still be durable — a promoted wisp, an + // issue retyped after it was created, or one written before its type + // joined types.infra — and while the narrowing held, `bd search --type + // message` (whose scope sets the flag) dropped exactly those rows with exit + // 0. IncludeInfra alone keeps the narrowing, and BuildCountFilter's + // IncludeInfra arm mirrors both halves (GH#4387). + if cfg.IsInfra(in.IssueType) && !in.IncludeEphemeral { + ephemeral := true + filter.Ephemeral = &ephemeral + } } // splitStatusSelector splits a --status selector into its trimmed diff --git a/internal/workapi/list_ephemeral_test.go b/internal/workapi/list_ephemeral_test.go index cb4fb0505..d1b11d49f 100644 --- a/internal/workapi/list_ephemeral_test.go +++ b/internal/workapi/list_ephemeral_test.go @@ -60,6 +60,50 @@ func TestIncludeEphemeralIsThePlaneBitAndNothingElse(t *testing.T) { } } +// TestIncludeEphemeralLiftsTheInfraTypeNarrowing pins the other half of the +// "admits in addition, never narrows" promise, for the one request that DOES +// narrow: naming an infra type reads only that type's ephemeral rows +// (Ephemeral=true), which is where creating one puts it. An infra-typed row can +// still be durable — a promoted wisp, a retyped issue — so IncludeEphemeral has +// to lift the narrowing, or `bd search --type message` (whose scope sets the +// flag) drops those rows with exit 0, as it did while the narrowing held. +// IncludeInfra alone keeps it; IncludeAllTypes, the union of the flags, lifts +// it with them. +func TestIncludeEphemeralLiftsTheInfraTypeNarrowing(t *testing.T) { + searchScope := issueops.SearchListRequest("needle") + searchScope.IssueType = "message" + + for _, tc := range []struct { + name string + req issueops.ListRequest + narrow bool + }{ + {"an infra type alone narrows to its ephemeral rows", issueops.ListRequest{IssueType: "message"}, true}, + {"IncludeInfra alone keeps the narrowing", issueops.ListRequest{IssueType: "message", IncludeInfra: true}, true}, + {"IncludeEphemeral lifts it", issueops.ListRequest{IssueType: "message", IncludeEphemeral: true}, false}, + {"IncludeEphemeral lifts it beside IncludeInfra", issueops.ListRequest{IssueType: "message", IncludeEphemeral: true, IncludeInfra: true}, false}, + {"IncludeAllTypes lifts it", issueops.ListRequest{IssueType: "message", IncludeAllTypes: true}, false}, + {"search's scope lifts it", searchScope, false}, + {"a type that is not infra is never narrowed", issueops.ListRequest{IssueType: "task"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + filter, err := BuildListFilter(tc.req, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + switch { + case tc.narrow && (filter.Ephemeral == nil || !*filter.Ephemeral): + t.Errorf("Ephemeral = %v, want true: the infra type's own wisps are the only rows this request asked for", filter.Ephemeral) + case !tc.narrow && filter.Ephemeral != nil: + t.Errorf("Ephemeral = %v, want nil: the durable rows of the type are part of this answer", *filter.Ephemeral) + } + if tc.req.IssueType == "message" && filter.SkipWisps { + t.Error("SkipWisps is set, but naming an infra type always reads the wisp table") + } + }) + } +} + // TestReadyProjectionCarriesTheEphemeralPlane pins the ReadyFlag arm's half of // the same promise. The projection onto WorkFilter is where a list field is // silently dropped, and a dropped IncludeEphemeral would answer `--ready diff --git a/internal/workapi/storereader/reader.go b/internal/workapi/storereader/reader.go index 8c0c2bae1..006b0deb0 100644 --- a/internal/workapi/storereader/reader.go +++ b/internal/workapi/storereader/reader.go @@ -114,7 +114,7 @@ func (r *storeReader) List(ctx context.Context, req issueops.ListRequest) (issue if req.ReadyFlag { items, err = r.store.GetReadyWorkWithCounts(ctx, workapi.ReadyFilterFromIssueFilter(filter)) } else { - items, err = r.store.SearchIssuesWithCounts(ctx, "", filter) + items, err = r.store.SearchIssuesWithCounts(ctx, req.Query, filter) } if err != nil { return issueops.IssuePage{}, err diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index d6905629c..6c46a786b 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "claimer.go", "commenter.go", "counter.go", + "current.go", "cycledetector.go", "deleter.go", "dependencyeditor.go", @@ -34,6 +35,7 @@ go_library( "readycounter.go", "relations.go", "releaser.go", + "search.go", "statsreporter.go", "sweeper.go", "template_errors.go", @@ -53,14 +55,17 @@ go_test( name = "issueops_test", srcs = [ "claimer_external_test.go", + "current_test.go", "issueops_external_test.go", "molecule_internal_test.go", "molecule_test.go", + "search_test.go", ], embed = [":issueops"], deps = [ "//:beads", "//internal/types", + "//internal/workapi", ], ) diff --git a/issueops/counter.go b/issueops/counter.go index 1f97939f4..f996cb5f4 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -167,7 +167,8 @@ type CountRequest struct { // - template molecules are excluded, which a default count includes; // - gate beads are excluded, unless IssueType asks for gates by name; // - an IssueType the workspace calls infra routes the count to the - // ephemeral tier instead of the durable one. + // ephemeral tier instead of the durable one — unless IncludeEphemeral + // is also set, which lifts that routing, as it does for a listing. // // The infra vocabulary is the WORKSPACE's, read from configuration inside // the implementation. A caller does not supply it and cannot: that is the @@ -198,6 +199,11 @@ type CountRequest struct { // it; only IncludeInfra sets out to reconcile cardinality, and it pays for // that with three changes beyond the plane. // + // Beside IncludeInfra it does one thing more, the same thing it does to a + // listing: it lifts the fourth change's routing of an infra IssueType, so + // the durable rows of that type (a promoted wisp, a retyped issue) are + // counted along with its wisps rather than instead of them. + // // Unset — and with IncludeInfra also unset — the count is durable-plane // only: the historical answer, unchanged. IncludeEphemeral bool diff --git a/issueops/current.go b/issueops/current.go new file mode 100644 index 000000000..30371185e --- /dev/null +++ b/issueops/current.go @@ -0,0 +1,64 @@ +package issueops + +import ( + "context" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/types" +) + +// CurrentIssueStatuses is the order FindCurrentIssue looks in: the actor's +// in-progress work first, then work hooked to it. +var CurrentIssueStatuses = []types.Status{types.StatusInProgress, types.StatusHooked} + +// FindCurrentIssue answers `bd show --current`'s question — which issue is +// this actor working on — through the Querier role, which every backend +// serves (a remote one as queryIssues). It returns the id of the first issue +// in the actor's in-progress work, else the first hooked to it, in the store's +// default order; "" when both reads answered and found nothing. +// +// A FAILED READ IS AN ERROR, never a miss. A caller falls back (to the +// last-touched issue, say) only on "", which only two reads that answered can +// produce. Treating a refusal as "nothing in progress" is what made the http +// backend print the wrong issue with exit 0. +// +// Each read is exactly the filter `status=`, `assignee=` and nothing +// else, one row. The actor travels as a quoted query value. The query language +// reads `assignee=none` and `assignee=null` (any case) as "no assignee", so an +// actor by one of those names, or an empty actor, cannot be asked about and +// gets "". +func FindCurrentIssue(ctx context.Context, querier Querier, actor string) (string, error) { + if querier == nil || actor == "" || strings.EqualFold(actor, "none") || strings.EqualFold(actor, "null") { + return "", nil + } + for _, status := range CurrentIssueStatuses { + page, err := querier.Query(ctx, currentIssueQuery(status, actor)) + if err != nil { + return "", fmt.Errorf("finding the current issue (%s, assigned to %s): %w", status, actor, err) + } + for _, row := range page.Items { + if row != nil && row.Issue != nil { + return row.ID, nil + } + } + } + return "", nil +} + +// currentIssueQuery is the role request for one status. No display order: +// every backend answers in its default order, and a sort would change which +// row is "current". +func currentIssueQuery(status types.Status, actor string) QueryRequest { + one := 1 + return QueryRequest{ + Expression: fmt.Sprintf("status=%s AND assignee=%s", status, quoteQueryValue(actor)), + Limit: &one, + } +} + +// quoteQueryValue renders v as a double-quoted query-language string, escaping +// the two characters the lexer gives meaning inside one. +func quoteQueryValue(v string) string { + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(v) + `"` +} diff --git a/issueops/current_test.go b/issueops/current_test.go new file mode 100644 index 000000000..d69170ff9 --- /dev/null +++ b/issueops/current_test.go @@ -0,0 +1,154 @@ +package issueops_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +type currentQueryResponse struct { + ids []string + err error +} + +type currentQueryRecorder struct { + responses []currentQueryResponse + calls []issueops.QueryRequest +} + +func (r *currentQueryRecorder) Query(_ context.Context, req issueops.QueryRequest) (issueops.IssuePage, error) { + r.calls = append(r.calls, req) + response := r.responses[len(r.calls)-1] + if response.err != nil { + return issueops.IssuePage{}, response.err + } + items := make([]*types.IssueWithCounts, 0, len(response.ids)) + for _, id := range response.ids { + items = append(items, &types.IssueWithCounts{Issue: &types.Issue{ID: id}}) + } + return issueops.IssuePage{Items: items}, nil +} + +func wantCurrentQueries(quotedActor string, statuses ...types.Status) []issueops.QueryRequest { + one := 1 + want := make([]issueops.QueryRequest, 0, len(statuses)) + for _, status := range statuses { + want = append(want, issueops.QueryRequest{ + Expression: "status=" + string(status) + " AND assignee=" + quotedActor, + Limit: &one, + }) + } + return want +} + +func TestFindCurrentIssueInProgressShortCircuits(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{ids: []string{"in-progress-first", "in-progress-second"}}}} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if err != nil || got != "in-progress-first" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (in-progress-first, nil)", got, err) + } + if want := wantCurrentQueries(`"tester"`, types.StatusInProgress); !reflect.DeepEqual(q.calls, want) { + t.Fatalf("Query calls = %#v, want %#v", q.calls, want) + } +} + +func TestFindCurrentIssueFindsHookedAfterInProgressMiss(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{}, {ids: []string{"hooked"}}}} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if err != nil || got != "hooked" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (hooked, nil)", got, err) + } + if want := wantCurrentQueries(`"tester"`, types.StatusInProgress, types.StatusHooked); !reflect.DeepEqual(q.calls, want) { + t.Fatalf("Query calls = %#v, want %#v", q.calls, want) + } +} + +func TestFindCurrentIssueAnswersEmptyAfterTwoMisses(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{}, {}}} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if err != nil || got != "" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (\"\", nil)", got, err) + } +} + +// TestFindCurrentIssueNeverTakesAFailureForAMiss is S6e: a refused or failed +// read used to fall through to the next read and then to the last-touched +// issue, so `bd show --current` printed the WRONG issue, exit 0. +func TestFindCurrentIssueNeverTakesAFailureForAMiss(t *testing.T) { + refusal := errors.New("SearchIssues: not supported over HTTP") + for _, tc := range []struct { + name string + responses []currentQueryResponse + calls int + }{ + {"in-progress read fails", []currentQueryResponse{{err: refusal}}, 1}, + {"hooked read fails", []currentQueryResponse{{}, {err: refusal}}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + q := ¤tQueryRecorder{responses: tc.responses} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if !errors.Is(err, refusal) || got != "" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (\"\", an error wrapping %v)", got, err, refusal) + } + if len(q.calls) != tc.calls { + t.Fatalf("query calls = %d, want %d", len(q.calls), tc.calls) + } + }) + } +} + +// TestFindCurrentIssueQuotesTheActor pins that an actor is one quoted value, +// whatever it contains, so it can neither break the parse nor splice in a +// clause. +func TestFindCurrentIssueQuotesTheActor(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{ids: []string{"x"}}}} + if _, err := issueops.FindCurrentIssue(context.Background(), q, `gastown/polecats/ru"st\ AND status=closed`); err != nil { + t.Fatal(err) + } + if want := wantCurrentQueries(`"gastown/polecats/ru\"st\\ AND status=closed"`, types.StatusInProgress); !reflect.DeepEqual(q.calls, want) { + t.Fatalf("Query calls = %#v, want %#v", q.calls, want) + } +} + +// TestFindCurrentIssueSkipsActorsItCannotAskAbout pins the actors the query +// language cannot express: `assignee=none`/`null` mean "no assignee", which +// would name some unassigned issue as this actor's. +func TestFindCurrentIssueSkipsActorsItCannotAskAbout(t *testing.T) { + for _, actor := range []string{"", "none", "NULL"} { + q := ¤tQueryRecorder{} + got, err := issueops.FindCurrentIssue(context.Background(), q, actor) + if err != nil || got != "" || len(q.calls) != 0 { + t.Fatalf("actor %q: got (%q, %v) after %d queries, want (\"\", nil) after none", actor, got, err, len(q.calls)) + } + } +} + +// TestFindCurrentIssueQueryMeansTheRawFilter pins that each read evaluates to +// exactly `status=`, `assignee=` — no predicate half, no default +// closed exclusion, no sort — so the store answers it with the same rows in +// the same default order the raw read did. +func TestFindCurrentIssueQueryMeansTheRawFilter(t *testing.T) { + for _, actor := range []string{"tester", "gastown/polecats/rust", `we"ird\name`, "has space"} { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{}, {}}} + if _, err := issueops.FindCurrentIssue(context.Background(), q, actor); err != nil { + t.Fatal(err) + } + for i, req := range q.calls { + plan, err := workapi.BuildQueryPlan(req) + if err != nil { + t.Fatalf("actor %q read %d: %v", actor, i, err) + } + f := plan.Filter + status := issueops.CurrentIssueStatuses[i] + if plan.RequiresPredicate() || f.Status == nil || *f.Status != status || f.Assignee == nil || *f.Assignee != actor || + f.NoAssignee || len(f.ExcludeStatus) != 0 || f.Limit != 1 || f.SortBy != "" { + t.Fatalf("actor %q status %s: plan = %+v", actor, status, plan) + } + } + } +} diff --git a/issueops/reader.go b/issueops/reader.go index dd761270c..852a9a9c2 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -152,6 +152,17 @@ type ListRequest struct { SpecPrefix string // IDFilter is a comma-separated id set. IDFilter string + // Query is `bd search`'s free text, with that command's meaning and not + // TitleSearch's: a row matches when its title contains the text or its id + // does, case-insensitively, and an id-shaped text also matches an exact + // id, an id prefix and the external ref. It is the `query` argument of the + // storage seam's SearchIssues, carried as a member so a backend that + // serves only roles can answer a search. + // + // It narrows; it lifts no default exclusion. A caller that wants search's + // "every status, every kind of row" answer asks for that with AllFlag and + // the Include* members, as for any other listing. + Query string Labels []string LabelsAny []string @@ -311,6 +322,14 @@ type ListRequest struct { // field alone reaches is the ephemeral rows of the types a listing already // shows. // + // AN INFRA IssueType IS THE ONE NARROWING IT LIFTS. Naming a type the + // workspace calls infra reads only that type's EPHEMERAL rows, which is + // where creating one puts it; with this flag set the request reads the + // type's rows on both planes instead, because an infra-typed row can still + // be durable — a promoted wisp, an issue retyped after it was created, or + // one written before its type joined types.infra. That is the superset + // promise above applied to the narrowing; IncludeInfra alone keeps it. + // // THE MERGED ANSWER IS ONE ORDER. Both planes are ordered together by // SortBy as if they were a single table — not the durable rows followed by // the ephemeral ones — so a Limit truncates the merged order rather than diff --git a/issueops/reader_ready_scope.go b/issueops/reader_ready_scope.go index bf3ee326c..b69bc686b 100644 --- a/issueops/reader_ready_scope.go +++ b/issueops/reader_ready_scope.go @@ -64,6 +64,7 @@ type readyScopeField struct { // that can never contain one — it is in the list below. var readyScopeFields = []readyScopeField{ {"IDFilter", "--id", func(r ListRequest) bool { return r.IDFilter != "" }}, + {"Query", "bd search", func(r ListRequest) bool { return r.Query != "" }}, {"TitleSearch", "--title", func(r ListRequest) bool { return r.TitleSearch != "" }}, {"SpecPrefix", "--spec", func(r ListRequest) bool { return r.SpecPrefix != "" }}, diff --git a/issueops/search.go b/issueops/search.go new file mode 100644 index 000000000..3739a10c7 --- /dev/null +++ b/issueops/search.go @@ -0,0 +1,33 @@ +package issueops + +// SearchListRequest is `bd search`'s request: the listing that answers the +// text query, with search's own scope. Every route of `bd search` — the direct +// store, the proxied server and a remote backend — asks issueops.Reader.List +// with this request (plus the caller's narrowing flags), so the three cannot +// answer the same search differently, and a remote backend serves it through +// the same role the server's listIssues handler calls. +// +// The scope is search's, not a listing's: +// - EVERY STATUS. Closed rows match unless the caller narrows with Status +// (bd-t5yex: "was this already filed or fixed?" must not answer no just +// because the answer is closed). AllFlag lifts the status exclusions and +// the pinned-flag predicate together. +// - EVERY KIND OF ROW. Templates, gates, the workspace's infrastructure types +// and the ephemeral plane all match; the four Include* members lift those +// exclusions. +// - THE STORE'S DEFAULT ORDER, priority then newest then id, so a Limit +// keeps the same rows a raw SearchIssues read kept. A display sort is the +// caller's to apply to the page afterwards. +// +// Limit is left nil (the listing default); a caller sets it. +func SearchListRequest(query string) ListRequest { + return ListRequest{ + Query: query, + AllFlag: true, + IncludeTemplates: true, + IncludeGates: true, + IncludeInfra: true, + IncludeEphemeral: true, + SortBy: "priority", + } +} diff --git a/issueops/search_test.go b/issueops/search_test.go new file mode 100644 index 000000000..9b19e8053 --- /dev/null +++ b/issueops/search_test.go @@ -0,0 +1,22 @@ +package issueops_test + +import ( + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestSearchListRequestIsSearchsScope pins the library's search scope: every +// status, every kind of row, the store's default order. +func TestSearchListRequestIsSearchsScope(t *testing.T) { + got := issueops.SearchListRequest("needle") + want := issueops.ListRequest{ + Query: "needle", AllFlag: true, + IncludeTemplates: true, IncludeGates: true, IncludeInfra: true, IncludeEphemeral: true, + SortBy: "priority", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("SearchListRequest = %+v, want %+v", got, want) + } +} From 85464f567aecc736059602dd0eab561bde7612b9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 13 Oct 2026 04:35:17 +0000 Subject: [PATCH 11/99] ci(bazel): dolt-race gates the dolt-server and proxied tiers in mode remote (item 5 A1b) (#7514) * ci(bazel): dolt-race gates the dolt-server and proxied tiers in mode remote (item 5 A1b) In mode remote, bazel-dolt-race (`bazel test //... --config=dolt-race`, the union of --config=doltserver and --config=doltserver-proxied) runs both tiers in place of bazel-doltserver and bazel-proxied, and is no longer advisory: its job.status is both the bazel-doltserver and the bazel-proxied workflow_call output (union result first, member's otherwise). In the fork modes, cache and local, the two member jobs run exactly as before: there one client does both tiers' work without in-flight deduplication, so a union would cost the sum of the tiers rather than the max. Exactly one arm runs in any mode, so gate ids, pr.yml, ci-gate.sh and bazel-gate.sh are unchanged. - bazel.yml: dolt-race if: mode == 'remote' (every event that reaches it); no continue-on-error; result output and always() recorder; test step timeout 25 -> 20 (job 55 >= step sum 46 + 5, < 60). Members' if: their old rule && mode != 'remote'. - scripts/bazel_dolt_race_test.go (was bazel_dolt_race_shadow_test.go): union-of-members config, sole-run lines in the union, gating shape, step and job timeouts, exactly one arm per mode x event with the call outputs evaluated as GitHub does, and the union's 62 test actions within --config=remote-exec's --jobs=64. - Gate simulations evaluate the workflow_call output expressions, and a gated lane's reporter is the lane or its union lane. - aquery: all 62 TestRunner actions under --config=dolt-race are byte-identical (key, argv, env, outputs) to --config=doltserver (with and without sole-run) and --config=doltserver-proxied on origin/main. - Docs: CI_REQUIRED_CHECK_TOPOLOGY.md "dolt-race Union Lane", TESTING.md. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): dolt-race test step 23 min so Bazel logs a 1200 s TIMEOUT first The proxied tier caps each test at 1200 s; a 20-minute step let GitHub kill a hung shard before Bazel wrote its TIMEOUT test.log and goroutine dump. Step 23, job 56 (step sum + 5, under the merge queue's 60). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): say the dolt-race job timeout bounds the job, not the run Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 25 +- .github/workflows/bazel.yml | 132 ++++--- .github/workflows/pr-risk.yml | 4 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 68 ++-- engdocs/TESTING.md | 4 +- scripts/BUILD.bazel | 2 +- scripts/bazel_cache_mode_test.go | 10 +- scripts/bazel_dolt_race_shadow_test.go | 305 --------------- scripts/bazel_dolt_race_test.go | 508 +++++++++++++++++++++++++ scripts/ci_analytics_workflow_test.go | 2 +- scripts/ci_blacksmith_runner_test.go | 6 +- scripts/ci_workflow_test.go | 128 ++++--- scripts/pr_risk_bazel_coverage_test.go | 12 +- 13 files changed, 734 insertions(+), 472 deletions(-) delete mode 100644 scripts/bazel_dolt_race_shadow_test.go create mode 100644 scripts/bazel_dolt_race_test.go diff --git a/.bazelrc b/.bazelrc index 90569a07d..2beb0d5be 100644 --- a/.bazelrc +++ b/.bazelrc @@ -178,20 +178,21 @@ test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 # above. test:doltserver-proxied --norun_validations -# dolt-server + proxied union (item 5, Phase 1 shadow): bazel.yml's advisory -# bazel-dolt-race job runs both tiers in one invocation, beside their own -# lanes, to measure one client per group of lanes before any cutover. Exactly -# the union of --config=doltserver's and --config=doltserver-proxied's lines, -# with one tag filter naming both tiers -# (scripts/bazel_dolt_race_shadow_test.go). Every other flag is the same in -# both members or outside any action key for the other: the race build, +# dolt-server + proxied union (item 5): bazel.yml's bazel-dolt-race job runs +# both tiers in one invocation in mode remote, in place of the bazel-doltserver +# and bazel-proxied lanes, which still run --config=doltserver and +# --config=doltserver-proxied in every other mode. Exactly the union of the two +# member configs' lines, with one tag filter naming both tiers +# (scripts/bazel_dolt_race_test.go). Every other flag is the same in both +# members or outside any action key for the other: the race build, # -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the eternal timeout # applies to no dolt-server target; the eviction retry count is what -# doltserver already gets from --config=sole-run in mode remote, the only mode -# the shadow runs in; test_summary, local_test_jobs and remote_download_regex -# are not in any action key. So each member target's test action is -# unchanged, and the two lanes' cached results carry over. The member configs -# stay for make, developers and the member lanes. +# doltserver gets from --config=sole-run wherever it executes remotely; +# test_summary, local_test_jobs and remote_download_regex are not in any action +# key. So each member target's test action is unchanged (aquery: all 62 +# TestRunner actions byte-identical), and whichever arm the mode picks shares +# cached results with the other. The member configs stay for those modes, make +# and developers. test:dolt-race --@rules_go//go/config:race test:dolt-race --test_tag_filters=dolt-server,dolt-server-proxied test:dolt-race --build_tests_only diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ba16b7f58..161a3bf74 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -46,11 +46,14 @@ # only what the PR changed is built and tested on the runner, and it is # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, -# workflow_dispatch, nightly.yml's call) gate nothing. bazel-dolt-race (the -# dolt-server + proxied union shadow, mode remote on PRs and merge groups -# only) cannot fail the gate either: no output, job-level continue-on-error. -# The gate still waits for it, as for every job of the call (see its comment -# and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Shadow"). +# workflow_dispatch, nightly.yml's call) gate nothing. In mode remote the +# dolt-server and proxied-server tiers run as one union job, bazel-dolt-race +# (`bazel test --config=dolt-race`), whose job.status is both the +# bazel-doltserver and the bazel-proxied output; in every other mode +# bazel-doltserver and bazel-proxied run as before and report their own. +# Exactly one of the two arms runs in any mode, so the gate ids +# (BAZEL_DOLTSERVER, BAZEL_PROXIED) and bazel-gate.sh's skips are unchanged +# (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Union Lane"). # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -346,12 +349,14 @@ on: bazel-integration: description: "bazel-integration's job.status" value: ${{ jobs.bazel-integration.outputs.result }} + # Mode remote: bazel-dolt-race runs both tiers and the member jobs + # skip; every other mode: the reverse. One side is always empty. bazel-doltserver: - description: "bazel-doltserver's job.status" - value: ${{ jobs.bazel-doltserver.outputs.result }} + description: "the dolt-server tier's job.status: bazel-dolt-race's in mode remote, bazel-doltserver's otherwise" + value: ${{ jobs.bazel-dolt-race.outputs.result || jobs.bazel-doltserver.outputs.result }} bazel-proxied: - description: "bazel-proxied's job.status" - value: ${{ jobs.bazel-proxied.outputs.result }} + description: "the proxied-server tier's job.status: bazel-dolt-race's in mode remote, bazel-proxied's otherwise" + value: ${{ jobs.bazel-dolt-race.outputs.result || jobs.bazel-proxied.outputs.result }} bazel-server-storage: description: "bazel-server-storage's job.status" value: ${{ jobs.bazel-server-storage.outputs.result }} @@ -1743,12 +1748,18 @@ jobs: # doctor/fix, protocol in 8 shards, testutil), each of which starts its # own dolt sql-server from the pinned dolt, so they execute on the remote # workers like any other test (locally with rbe=off). + # Not in mode remote: there bazel-dolt-race runs this tier together with + # the proxied-server tier and reports for both (item 5). In the fork + # modes, cache and local there is no in-flight deduplication of one + # client's actions and the client does the work, so a union would cost the + # sum of the two tiers instead of the max: this job runs there as before. bazel-doltserver: name: dolt-server lane - # Same runner and skip rule as bazel-test. + # Same runner as bazel-test; bazel-test's skip rule, plus mode remote + # (bazel-dolt-race's). needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }} - if: ${{ needs.rbe.outputs.mode != 'skip' }} + if: ${{ needs.rbe.outputs.mode != 'skip' && needs.rbe.outputs.mode != 'remote' }} timeout-minutes: 90 outputs: result: ${{ steps.result.outputs.result }} @@ -1879,7 +1890,8 @@ jobs: # (formerly proxied-local-smoke.yml) in a loopback-only network # namespace, which needs the workers' unprivileged user namespaces. A job # of its own, not gated by any integration switch: it runs in pr.yml's - # call and its gate requires it (BAZEL_PROXIED). + # call and its gate requires it (BAZEL_PROXIED). Fork modes only: in mode + # remote bazel-dolt-race runs this tier (see bazel-doltserver's comment). bazel-proxied: name: proxied-server tier # Remote execution only, like bazel-embedded: 30 concurrent shards, each @@ -1889,7 +1901,7 @@ jobs: # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' }} timeout-minutes: 30 outputs: result: ${{ steps.result.outputs.result }} @@ -2033,45 +2045,37 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" - # Item 5, Phase 1 (scripts/bazel_dolt_race_shadow_test.go): the advisory - # shadow of a union lane. One `bazel test --config=dolt-race` runs the - # dolt-server and proxied-server tiers together (.bazelrc: exactly the - # union of --config=doltserver's and --config=doltserver-proxied's lines, - # one tag filter for both), beside bazel-doltserver and bazel-proxied, so - # one client per group of lanes can be compared with the two it would - # replace (end time, runner minutes, test counts and results) before any - # cutover. Each member target's test action key is unchanged, so the shadow - # and the member lanes share cached results; and rbe-west deduplicates - # identical in-flight executions (confirmed by the farm maintainer: - # NativeLink 1.7.1 subscribes a second identical Execute of a cacheable - # action to the operation already in flight), so the shadow adds no - # executions. - # Advisory: it cannot fail the gate. Job-level continue-on-error (a failed - # job otherwise fails the whole call, whose result pr.yml's ci-gate - # requires as BAZEL), no workflow_call output, no job outputs, no job in - # this file needs it, and no gate script or caller names it. The gate does - # wait for it: ci-gate needs pr.yml's bazel call, which completes only when - # every job in it has finished (success, failure or timeout), so a slow - # shadow delays the gate and the merge queue. continue-on-error does not - # cover a job GitHub cancels at its own timeout-minutes, which could turn - # the call red: every step has a timeout, and the job's exceeds their sum - # (TestBazelDoltRaceShadowIsAdvisory). Mode remote only, on - # same-repo pull requests and merge groups (mode remote on a pull_request - # is never a fork): no fork mode, no cache or local mode, and not on push, - # nightly, dispatch or bazel-farm.yml's call. Its ci-analytics LANE is - # dolt-race. Removing it is the rollback; nothing reads it. + # Item 5 (scripts/bazel_dolt_race_test.go): the dolt-server and + # proxied-server tiers in one `bazel test --config=dolt-race` (.bazelrc: + # exactly the union of --config=doltserver's and + # --config=doltserver-proxied's lines, one tag filter for both), mode + # remote only, in place of bazel-doltserver and bazel-proxied, which run + # in every other mode. One client waits on rbe-west for both tiers' + # shards at once (27 + 35 test actions, under the remote-exec config's 64 + # jobs), so the job ends with the slower tier and saves the other's + # runner. Each member target's test action key is unchanged (aquery- + # verified), so cached results carry over both ways, and a revert of the + # cutover re-executes nothing. Gated: its job.status is both the + # bazel-doltserver and the bazel-proxied workflow_call output (a failure + # in either tier fails both ids, which are both required). Every step has + # a timeout and the job's exceeds their sum by 5 minutes while staying + # under the merge queue's 60-minute check timeout + # (TestBazelDoltRaceJobTimeouts), so a hang fails a named step instead of + # dropping the merge group. The member jobs' configs and the union's must + # stay key-identical: the mode decides which runs (TestBazelDoltRaceConfigIsMemberUnion). + # Its ci-analytics LANE is dolt-race. bazel-dolt-race: name: dolt-race needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} - # At least every step's timeout-minutes plus 5 (and the test step's plus - # 10): a hung step then fails the job (continue-on-error) before GitHub - # cancels it, which continue-on-error does not cover - # (TestBazelDoltRaceShadowIsAdvisory). bazel-proxied's steps have no - # timeouts but its test step's; these bound the rest. - timeout-minutes: 55 - continue-on-error: true + if: ${{ needs.rbe.outputs.mode == 'remote' }} + # At least every step's timeout-minutes plus 5, and below the merge + # queue's 60-minute check timeout so this job alone cannot exhaust it + # (TestBazelDoltRaceJobTimeouts). Not a whole-run bound: the rbe job runs + # first and sibling lanes have longer timeouts. + timeout-minutes: 56 + outputs: + result: ${{ steps.result.outputs.result }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -2120,11 +2124,15 @@ jobs: fi # Both tiers' shards at once: 27 dolt-server + 35 proxied test actions, - # under the remote-exec config's 64 jobs. The step timeout is - # bazel-proxied's (the longer tier on rbe-west). + # under the remote-exec config's 64 jobs. No --config=sole-run: its one + # line (no retry on a remote cache eviction) is already in the union. + # Shards are expected around 55-95s on rbe-west (see bazel-proxied's + # test step), the proxied tier's 1200s cap included. The step timeout is + # >= proxied test_timeout 1200 s + about 3 min so Bazel writes TIMEOUT + # logs before GitHub kills the step. - name: bazel test //... --config=dolt-race id: test - timeout-minutes: 25 + timeout-minutes: 23 run: | set -o pipefail start=$(date +%s) @@ -2138,12 +2146,11 @@ jobs: echo "bazel test --config=dolt-race: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" - # The member lanes' checks over the union's BEP: bazel-proxied's (the - # dolt-server lane has none). check_shard_coverage ignores targets it - # does not list; check_testcases checks every target, so it covers the - # dolt-server tier too, which bazel-doltserver does not check: a - # dolt-server shard that runs no test turns this job red while both - # member lanes stay green. + # The member lanes' checks, once over the union's BEP: bazel-proxied's + # (the dolt-server lane has none). check_shard_coverage ignores targets + # it does not list; check_testcases checks every target, so it covers + # the dolt-server tier too, which bazel-doltserver does not check: a + # dolt-server shard that runs no test turns this job red. - name: Every target and shard ran tests if: ${{ always() && steps.test.outcome != 'skipped' }} timeout-minutes: 2 @@ -2201,6 +2208,17 @@ jobs: retention-days: 7 if-no-files-found: ignore + # Last step: the job's result as the bazel-doltserver and + # bazel-proxied workflow_call outputs for pr.yml's ci-gate. always() + # runs it after a failure, a timeout or a cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + timeout-minutes: 1 + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: # the conformance and 16-shard full-suite variants of diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index bf7960b75..155b3b944 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -24,8 +24,8 @@ env: # The legacy Go test tiers this workflow used to run (embedded Dolt storage, # conformance and cmd shards; proxied-server cmd shards; server-Dolt storage # and full suite) are pr.yml's Bazel lanes on rbe-west now: `embedded Dolt -# tier`, `proxied-server tier` and `server-Dolt storage tier` (bazel.yml), -# which pr.yml's `CI Gate / Required` requires to have run remotely and +# tier`, `proxied-server tier` (`dolt-race`, with the dolt-server lane, in +# mode remote) and `server-Dolt storage tier` (bazel.yml), which pr.yml's `CI Gate / Required` requires to have run remotely and # passed on every pull_request and merge group, fork and Dependabot PRs # included (ga-96smfk.22). What remains is the Nix packaging check, which no # Bazel target can stand in for: it builds the package Nix users install. diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 333b30e29..6590acea7 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -54,7 +54,10 @@ Current PR-related workflow names: repo variable is unset) and exports it as the `rbe-mode` / `rbe-enabled` outputs (`rbe-enabled` is `true` in `remote`, `fork-ro` and `fork-rw`); every lane exports its - `job.status` as an output named after the job. `pr.yml`'s gate requires the + `job.status` as an output named after the job (in mode `remote`, the + `bazel-doltserver` and `bazel-proxied` outputs carry the `bazel-dolt-race` + union job's status instead; see [dolt-race Union Lane](#dolt-race-union-lane)). + `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_RELEASE_CROSS`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go @@ -620,38 +623,41 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. -## dolt-race Shadow (Not Required) +## dolt-race Union Lane -`bazel.yml`'s `bazel-dolt-race` job (check name `Bazel / dolt-race`) is an -advisory shadow, not a required check. It runs -`bazel test //... --config=dolt-race`, the union of `--config=doltserver` and -`--config=doltserver-proxied` with one tag filter for both tiers, beside the -`bazel-doltserver` and `bazel-proxied` lanes. Its purpose is to measure one -client per group of lanes against the two lanes it would replace (end time, -runner minutes, test counts and results) before any cutover. Each member -target's test action key is unchanged, so the shadow shares cached results -with the member lanes. +In mode `remote`, `bazel.yml`'s `bazel-dolt-race` job (check name +`Bazel / dolt-race`) runs the dolt-server and proxied-server tiers in one +`bazel test //... --config=dolt-race`, in place of the `bazel-doltserver` +(`dolt-server lane`) and `bazel-proxied` (`proxied-server tier`) jobs. +`--config=dolt-race` is exactly the union of `--config=doltserver` and +`--config=doltserver-proxied`, with one tag filter for both tiers, so every +member target's test action is unchanged (`bazel aquery`: the 27 + 35 = 62 +`TestRunner` actions are byte-identical under the member configs and the +union). One client waits on rbe-west for both tiers at once, so the job ends +with the slower tier and the other tier's runner is saved. -- It runs in mode `remote` on `pull_request` and `merge_group` events only. - It never runs in a fork mode (`fork-ro`, `fork-rw`), in mode `cache` or - `local`, on `bazel-farm.yml`'s call, on push, nightly or dispatch. -- It cannot fail the gate. It has job-level `continue-on-error`, so its - failure cannot fail the call's aggregate result (`BAZEL`). It has no - `workflow_call` output, no `bazel.yml` job `needs` it, and `pr.yml`, - `ci-gate.sh` and `bazel-gate.sh` never name it. -- The gate still waits for it. `CI Gate` needs `pr.yml`'s `bazel` call, and - a reusable-workflow call completes only when all of its jobs have finished - (success, failure or timeout), `continue-on-error` ones included. A slow - shadow therefore delays `CI Gate` and the merge queue, up to its - timeout. `continue-on-error` does not cover a job GitHub cancels at its - own `timeout-minutes`, so every step has its own timeout and the job's - (65 minutes) stays at least 5 minutes above their sum and 10 above the - test step's (policy-tested). A hung step then fails the job, which - `continue-on-error` covers, before GitHub would cancel it. +- Mode `remote` only (same-repo pull requests, merge groups, push, nightly, + dispatch and `bazel-farm.yml`'s authorized call). In the fork modes + (`fork-ro`, `fork-rw`), `cache` and `local`, `bazel-doltserver` and + `bazel-proxied` run as before: there a single client does both tiers' work + without in-flight deduplication, so a union would cost the sum of the + tiers rather than the max, on the run's critical path. +- Exactly one arm runs in any mode. The union's `job.status` is both the + `bazel-doltserver` and the `bazel-proxied` `workflow_call` output (each + reads the union's result first, then its member's), so `pr.yml`'s gate + ids (`BAZEL_DOLTSERVER`, `BAZEL_PROXIED`), `ci-gate.sh` and + `bazel-gate.sh` are unchanged: a failed or cancelled union reds both ids, + a union that reported nothing reads as `skipped`, which mode `remote` + never allows, and the per-mode skips are the members' as before. +- `check_testcases.py` and `check_shard_coverage.py` run once over the + union's BEP (the proxied tier's `--suite`; the dolt-server tier has no + shard-coverage check). The `bazel test` step has a 20-minute timeout, + every step has one, and the job's (55 minutes) is at least their sum plus + 5 and below the merge queue's 60-minute check timeout. - Its ci-analytics `LANE` is `dolt-race`. -- Policy: `scripts/bazel_dolt_race_shadow_test.go`. Removing the job, its - `.bazelrc` config and its line in `.github/scripts/rrc-lane-commands.txt` - is the rollback; nothing reads them. +- Policy: `scripts/bazel_dolt_race_test.go`. Rollback is a revert of the + cutover PR: the gate ids never changed and the action keys are equal both + ways, so cached results carry back. ## Remote Repo Contents Cache @@ -1032,7 +1038,7 @@ Required` requires them to have run remotely and passed. `nogoConfigurations` table (race, pure, js/wasm) validates in exactly one required lane, not beside every compile of every lane: the race configuration's nogo is owned by `test` above, and `embedded`, - `doltserver`, `doltserver-proxied`, `dolt-race` (shadow) and the + `doltserver`, `doltserver-proxied`, `dolt-race` and the package gates pass `--norun_validations`, since they compile a strict subset of what `test` already validates on that configuration (`scripts/nogo_lint_policy_test.go`, engdocs/LINTING.md). Since F5 S2 diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 595887d92..8e64bd0ca 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -43,6 +43,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | +| Dolt server + proxied (`bazel-dolt-race`) | `bazel test //... --config=dolt-race` | Both rows above in one invocation; CI runs this instead of them in mode `remote`. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -199,7 +200,8 @@ remotely with `--config=remote-exec`. No Bazel target uses the `container` backend; run `go test` with docker and the pulled image to exercise it. PR Risk's heavier server tiers have configs of their own, run by bazel.yml only with remote execution, each in a job of its own (`bazel-proxied`, -`bazel-server-storage`): `--config=doltserver-proxied` is the +which in mode `remote` is folded with `bazel-doltserver` into +`bazel-dolt-race`, and `bazel-server-storage`): `--config=doltserver-proxied` is the proxied-server cmd/bd tier ("Test (Proxied Dolt Cmd N/15)", `//cmd/bd:bd_proxied_test`), and `--config=doltserver-integration` the server-Dolt storage tier ("Test (Server Dolt Conformance)", "Test (Server diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 97a16d586..f8e72de2c 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -35,7 +35,7 @@ go_test( srcs = [ "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", - "bazel_dolt_race_shadow_test.go", + "bazel_dolt_race_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", "bazel_go_repository_tools_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 437910fde..9b1b3152e 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -227,9 +227,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // no longer forwards the app secrets either, so there is no // audience left for pre-warming in fork-ro or fork-rw (see // bazel.yml's comment on the job). - // Nor does a shadow lane (bazelShadowLanes): mode remote on - // same-repo PRs and merge groups only. - case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelShadowLanes[name] == "": + // Nor does a union lane (bazelUnionLanes): mode remote only, + // its members run in the fork modes. + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelUnionLanes[name] == nil: t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && (name == bazelIntegJobName || name == bazelCmdDoltJobName) && !runs: t.Errorf("%s does not run in mode cache", name) @@ -289,7 +289,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { if bazelPackageJobs[lane] { continue } - if !ran[lane] { + // A union lane's run reports for its members + // (bazel_dolt_race_test.go). + if !ran[lane] && !ran[bazelUnionOf(lane)] { wantSkips = append(wantSkips, id) } } diff --git a/scripts/bazel_dolt_race_shadow_test.go b/scripts/bazel_dolt_race_shadow_test.go deleted file mode 100644 index 0b6008047..000000000 --- a/scripts/bazel_dolt_race_shadow_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package scripts_test - -import ( - "cmp" - "os" - "path/filepath" - "slices" - "strings" - "testing" - - "gopkg.in/yaml.v3" -) - -// The dolt-race shadow (item 5, Phase 1): bazel.yml's bazel-dolt-race job -// runs --config=dolt-race, the union of --config=doltserver and -// --config=doltserver-proxied, in one `bazel test` beside the two member -// lanes, so the union-job model can be measured against them before any -// cutover. It must be advisory: no workflow_call output, nothing in pr.yml's -// gates or bazel-gate.sh/ci-gate.sh, and a failure that cannot fail the -// call's aggregate result (BAZEL) that pr.yml's ci-gate requires. It cannot -// fail the gate, but the gate still waits for it: ci-gate needs the whole -// bazel call, which completes only when every job in it has finished. - -const ( - bazelDoltRaceJobName = "bazel-dolt-race" - bazelDoltRaceConfig = "dolt-race" - // Mode remote only, and only on same-repo pull requests and merge - // groups (mode remote on a pull_request event is never a fork: forks - // get fork-ro, fork-rw or cache). Not on push, nightly, dispatch or - // bazel-farm.yml's pull_request_target call. - bazelDoltRaceJobIf = "${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}" -) - -// bazelShadowLanes: bazel.yml jobs that run beside gated lanes only to -// measure a change to them, and report into nothing (no job or -// workflow_call output, no gate id, job-level continue-on-error), with the -// reason. TestBazelLaneIsGated and the continue-on-error checks carve them -// out by this map. -var bazelShadowLanes = map[string]string{ - bazelDoltRaceJobName: "item 5 Phase 1: the dolt-server + proxied union beside its member lanes, advisory until a cutover PR", -} - -// bazelDoltRaceJobTimeoutHeadroom: the minutes the shadow job's -// timeout-minutes must exceed its test step's (TestBazelDoltRaceShadowIsAdvisory). -const bazelDoltRaceJobTimeoutHeadroom = 10 - -// bazelDoltRaceJobTimeoutMargin: the minutes the shadow job's timeout-minutes -// must exceed the sum of all its step timeouts. -const bazelDoltRaceJobTimeoutMargin = 5 - -// bazelDoltRaceMergeQueueTimeout: the merge queue's check timeout (ruleset -// merge_queue.check_response_timeout_minutes). CI Gate waits for the shadow, -// so the shadow's job timeout must stay below it or a hung shadow drops the -// group from the queue. -const bazelDoltRaceMergeQueueTimeout = 60 - -// The union's members, and the tag filter each owns. -var bazelDoltRaceMembers = []struct{ config, tag string }{ - {"doltserver", "dolt-server"}, - {"doltserver-proxied", "dolt-server-proxied"}, -} - -// bazelRCConfigLines returns .bazelrc's option lines for --config=name -// (any command: test:name, build:name, ...), as "