From 55aac471761c69704551bdecda67e32b9b9fef3d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 13 Oct 2026 15:43:37 +0000 Subject: [PATCH 01/76] test(bazel): pin the item-5 one-wave rule for dolt-race/dolt-int budgets (#7545) * test(bazel): pin the item-5 one-wave rule for dolt-race and dolt-int budgets Closed PR #7542 found that a union with more test actions than its effective --jobs can regress wall clock by up to +575s, because Bazel does not schedule the longest action first and a second-wave test adds its full duration to the critical path instead of overlapping it. Compute the effective --jobs from .bazelrc (remote-exec's baseline, overridden by a --jobs line on the union's own config, last one winning, matching Bazel's BEP cmdLine semantics from #7542) instead of reading remote-exec's line alone, and explain the second-wave tail in the budget tests' failure messages. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): clarify why the union's --jobs wins (comment only) Command-line config outranks rc-file config; the BEP cmdLine ordering PR #7542 observed is a consequence of that rank, not "last one wins" by position, so reordering .bazelrc's lines cannot change precedence. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- scripts/bazel_dolt_int_test.go | 24 ++++++++------- scripts/bazel_dolt_race_test.go | 53 ++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/scripts/bazel_dolt_int_test.go b/scripts/bazel_dolt_int_test.go index 25bf12313..c961fab73 100644 --- a/scripts/bazel_dolt_int_test.go +++ b/scripts/bazel_dolt_int_test.go @@ -528,19 +528,21 @@ func TestCheckTestcasesUnitTests(t *testing.T) { // cmd-dolt (aquery, item 5 A2). const bazelDoltIntTestActions = 63 -// TestBazelDoltIntFitsTheJobsBudget: the union runs both tiers' shards in one -// invocation under --config=remote-exec's --jobs, so all of them must fit at -// once or the slower tier's shards queue behind the other's. Every rule -// tagged with one of the union's tags, read from the BUILD files: its -// shard_count (1 when unsharded), summed. +// TestBazelDoltIntFitsTheJobsBudget pins the one-wave rule (closed PR #7542, +// carried from dolt-race: see TestBazelDoltRaceFitsTheJobsBudget's own +// comment for the mechanism and the measured +575s regression): a union's +// total test actions must fit in the effective --jobs +// (bazelUnionEffectiveJobs) so every action starts in Bazel's first wave, +// since Bazel does not schedule the longest action first and a second-wave +// tail test adds its own full duration to the critical path instead of +// overlapping it. The union runs both tiers' shards in one invocation under +// that budget, so all of them must fit at once or the slower tier's shards +// queue behind the other's. Every rule tagged with one of the union's tags, +// read from the BUILD files: its shard_count (1 when unsharded), summed. func TestBazelDoltIntFitsTheJobsBudget(t *testing.T) { root := bazelPolicyRoot(t) rc := readPolicyFile(t, root, ".bazelrc") - m := regexp.MustCompile(`(?m)^build:remote-exec --jobs=(\d+)$`).FindStringSubmatch(rc) - if m == nil { - t.Fatal(".bazelrc has no build:remote-exec --jobs=N line") - } - budget, _ := strconv.Atoi(m[1]) + budget := bazelUnionEffectiveJobs(t, rc, bazelDoltIntConfig) var tags []string for _, mem := range bazelDoltIntMembers { @@ -587,7 +589,7 @@ func TestBazelDoltIntFitsTheJobsBudget(t *testing.T) { t.Errorf("--config=%s: %d test actions over %d targets, want %d (aquery-verified at the cutover): re-check against --jobs=%d and update", bazelDoltIntConfig, total, targets, bazelDoltIntTestActions, budget) } if total > budget { - t.Errorf("--config=%s: %d test actions, more than --config=remote-exec's --jobs=%d: the tiers would no longer run at once", bazelDoltIntConfig, total, budget) + t.Errorf("--config=%s: %d test actions, more than the effective --jobs=%d: a second wave would add its tail test's full duration to the critical path instead of overlapping it (closed PR #7542 measured up to +575s from exactly this)", bazelDoltIntConfig, total, budget) } } diff --git a/scripts/bazel_dolt_race_test.go b/scripts/bazel_dolt_race_test.go index 1225344ec..ef0ae2d86 100644 --- a/scripts/bazel_dolt_race_test.go +++ b/scripts/bazel_dolt_race_test.go @@ -186,6 +186,40 @@ func bazelRCConfigLines(rc, name string) []string { return out } +// bazelUnionEffectiveJobs: the --jobs value in effect when a union job's +// `bazel test //... --config=` step runs in mode remote +// (closed PR #7542). Mode remote's generated rc applies +// `build --config=remote-exec` from an rc file, so --config=remote-exec's +// --jobs line is the baseline every union inherits; but a --jobs line on +// the union's own config outranks it, because that --config is given on +// the command line (the step's own `--config=`) and +// command-line config outranks rc-file config regardless of which file's +// line comes first (PR #7542's BEP showed `optionsParsed.cmdLine` carrying +// "--jobs=64, --jobs=128" in that rank order, after a union config added +// its own --jobs=128 override; the ordering there is a consequence of the +// rank, not the rule itself, so reordering .bazelrc's lines cannot change +// it). Reading remote-exec's line alone would miss that override silently +// passing a budget check the real run does not meet. +func bazelUnionEffectiveJobs(t *testing.T, rc, unionConfig string) int { + t.Helper() + base := regexp.MustCompile(`(?m)^build:remote-exec --jobs=(\d+)$`).FindStringSubmatch(rc) + if base == nil { + t.Fatal(".bazelrc has no build:remote-exec --jobs=N line") + } + budget, _ := strconv.Atoi(base[1]) + jobsLine := regexp.MustCompile(`^--jobs=(\d+)$`) + for _, l := range bazelRCConfigLines(rc, unionConfig) { + _, opt, ok := strings.Cut(l, " ") + if !ok { + continue + } + if m := jobsLine.FindStringSubmatch(opt); m != nil { + budget, _ = strconv.Atoi(m[1]) + } + } + return budget +} + // TestBazelDoltRaceConfigIsMemberUnion: --config=dolt-race is exactly the // set union of its members' lines, except that each member's // --test_tag_filters is replaced by one filter naming both members' tags. @@ -472,8 +506,15 @@ func TestBazelDoltRaceArmsExactlyOnePerMode(t *testing.T) { // proxied-server (aquery, item 5 A1b), each a target's shard. const bazelDoltRaceTestActions = 62 -// TestBazelDoltRaceFitsTheJobsBudget: the union runs both tiers' shards in -// one invocation under --config=remote-exec's --jobs, so all of them must +// TestBazelDoltRaceFitsTheJobsBudget pins the one-wave rule (closed PR +// #7542): a union's total test actions must fit in the effective --jobs so +// every action starts in Bazel's first wave. Bazel does not schedule the +// longest action first, so an action that lands in a second wave because +// the first is full adds its own full duration to the critical path rather +// than overlapping it; #7542's paired A/B runs measured this costing up to +// +575 s of wall clock on a union that exceeded its --jobs by one tier's +// worth of actions. The union runs both tiers' shards in one invocation +// under the effective --jobs (bazelUnionEffectiveJobs), so all of them must // fit at once or the slower tier's shards queue behind the other's and the // union stops ending with the slower member. Every rule tagged with one of // the union's tags, read from the BUILD files: its shard_count (1 when @@ -482,11 +523,7 @@ const bazelDoltRaceTestActions = 62 func TestBazelDoltRaceFitsTheJobsBudget(t *testing.T) { root := bazelPolicyRoot(t) rc := readPolicyFile(t, root, ".bazelrc") - m := regexp.MustCompile(`(?m)^build:remote-exec --jobs=(\d+)$`).FindStringSubmatch(rc) - if m == nil { - t.Fatal(".bazelrc has no build:remote-exec --jobs=N line") - } - budget, _ := strconv.Atoi(m[1]) + budget := bazelUnionEffectiveJobs(t, rc, bazelDoltRaceConfig) var tags []string for _, mem := range bazelDoltRaceMembers { @@ -533,6 +570,6 @@ func TestBazelDoltRaceFitsTheJobsBudget(t *testing.T) { t.Errorf("--config=%s: %d test actions over %d targets, want %d (aquery-verified at the cutover): re-check against --jobs=%d and update", bazelDoltRaceConfig, total, targets, bazelDoltRaceTestActions, budget) } if total > budget { - t.Errorf("--config=%s: %d test actions, more than --config=remote-exec's --jobs=%d: the tiers would no longer run at once", bazelDoltRaceConfig, total, budget) + t.Errorf("--config=%s: %d test actions, more than the effective --jobs=%d: a second wave would add its tail test's full duration to the critical path instead of overlapping it (closed PR #7542 measured up to +575s from exactly this)", bazelDoltRaceConfig, total, budget) } } From d4ca89369f72e4666fe76493e4cab700a7e73320 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 13 Oct 2026 23:13:29 +0000 Subject: [PATCH 02/76] fix(http): bd gate list over http through issueops.ListGates (S20) (#7549) The workspace-wide gate listing built a raw SearchIssues filter, which the HTTP backend refuses (B3 #15), so gc's renudge-stale-human-gates order (gc bd gate list --limit 0 --json) failed on every remote scope. issueops.ListGates asks the Querier role (queryIssues, served by every deployed server) for type=gate, closed hidden unless --all: the rows the raw filter selected, on both planes. The direct, proxied and remote routes all call it through openQuerier. The corpus seed gains an open, a closed and an ephemeral gate, and mc-gate-list (the order's argv) plus mc-gate-list-all pass on the served side. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 10 +++ backend/http/corpus/testdata/corpus.yaml | 15 +++-- cmd/bd/gate.go | 43 ++++++------- cmd/bd/gate_proxied_server.go | 65 ++++++++----------- issueops/BUILD.bazel | 2 + issueops/gates.go | 58 +++++++++++++++++ issueops/gates_test.go | 80 ++++++++++++++++++++++++ 7 files changed, 208 insertions(+), 65 deletions(-) create mode 100644 issueops/gates.go create mode 100644 issueops/gates_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a610fa8..c2b1f1629 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -439,6 +439,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `RunBatchApplyAppliesTheDefaultPriority`. ### Fixed + +- **`bd gate list` works over a connected HTTP workspace** (S20). The + workspace-wide listing built a raw `SearchIssues` filter, which the HTTP + backend refuses, so gc's `renudge-stale-human-gates` order failed on every + remote scope. Every route now calls one library entry, + `issueops.ListGates`, which asks the Querier role (`queryIssues`, served by + every deployed server) for `type=gate` with closed rows hidden unless + `--all`: the same rows the raw filter selected, on both planes. `--limit 0` + is unlimited, as before. The bead-scoped form (`bd gate list `) is + unchanged. - **An update of a template is now refused for every caller, not only `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 10c43ba34..532103c1c 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -54,6 +54,9 @@ files: {"key": "wisp-p", "id": "e2e-wisp-p", "title": "Wisp wisp-p", "type": "task", "ephemeral": true}, {"key": "wisp-m", "id": "e2e-wisp-m", "title": "Wisp molecule", "type": "epic", "ephemeral": true}, {"key": "wisp-m.1", "id": "e2e-wisp-m.1", "title": "wisp step", "type": "task", "ephemeral": true, "parent_key": "wisp-m"}, + {"key": "gate-o", "id": "e2e-gate-o", "title": "Open gate", "type": "gate"}, + {"key": "gate-c", "id": "e2e-gate-c", "title": "Closed gate", "type": "gate"}, + {"key": "wisp-gate", "id": "e2e-wisp-gate", "title": "Wisp gate", "type": "gate", "ephemeral": true}, {"key": "t-upd", "id": "e2e-t-upd", "title": "target upd", "type": "task", "labels": ["lane:z"]}, {"key": "t-upd2", "id": "e2e-t-upd2", "title": "target upd2", "type": "task"}, {"key": "t-m1", "id": "e2e-t-m1", "title": "target m1", "type": "task"}, @@ -136,7 +139,7 @@ seed: # (gc's own plans never set a status). Lifecycle goes through update/close. - [update, e2e-ip1, e2e-ipna, e2e-ipnb, e2e-mol.1, e2e-mac.2, e2e-t-rel, e2e-t-xfer, e2e-t-ifa, e2e-t-death, e2e-t-rc, e2e-t-boot, --status, in_progress] - [update, e2e-wisp-2, --status, in_progress] - - [close, --force, e2e-done, e2e-ep.2, e2e-mol.3, e2e-mac.1, e2e-t-ro] + - [close, --force, e2e-done, e2e-ep.2, e2e-mol.3, e2e-mac.1, e2e-t-ro, e2e-gate-c] - [comments, add, e2e-r1, "seed comment"] shapes: @@ -418,10 +421,14 @@ shapes: source: gc checks_proxied_backup_coverage.go (backup status --json) args: [backup, status, --json] compare: exit + # The seed holds an open gate, a closed gate and an ephemeral gate, so the + # open listing and the --all listing answer different, non-empty sets. - id: mc-gate-list - source: core pack orders (gate list) - args: [gate, list, --json] - pending: {slice: "S20", reason: "gate list issues a raw SearchIssues shape the bridge refuses (B3 #15)"} + source: gc core pack order renudge-stale-human-gates.sh (gc bd gate list --limit 0 --json) + args: [gate, list, --limit, "0", --json] + - id: mc-gate-list-all + source: core pack orders (gate list --all) + args: [gate, list, --all, --json] - id: gc-export-all source: gc core pack reaper (gc bd export --all -o F) diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 98985e062..d42e2386d 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -17,6 +17,7 @@ import ( storageissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) // gateCmd is the parent command for gate operations @@ -100,30 +101,30 @@ By default, shows only open gates. Use --all to include closed gates.`, return nil } - gateType := types.IssueType("gate") - filter := types.IssueFilter{ - IssueType: &gateType, - Limit: limit, - } - - if !allFlag { - filter.ExcludeStatus = []types.Status{types.StatusClosed} - } - - issues, err := store.SearchIssues(ctx, "", filter) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - if jsonOutput { - return outputJSON(issues) - } - - displayGates(issues, allFlag) - return nil + return runGateList(ctx, allFlag, limit) }, } +// runGateList lists the workspace's gates on every route through ONE library +// entry, issueops.ListGates, over the route's Querier (openQuerier picks the +// direct store, the proxied server's unit of work, or the remote http +// backend). Only the presentation stays here. +func runGateList(ctx context.Context, allFlag bool, limit int) error { + querier, err := openQuerier() + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + gates, err := issueops.ListGates(ctx, querier, issueops.GateListRequest{All: allFlag, Limit: limit}) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + if jsonOutput { + return outputJSON(gates) + } + displayGates(gates, allFlag) + return nil +} + // filterIssueGates selects the gate-type issues from an issue's dependency set, // honoring the same open/closed and limit semantics as the DB-wide list path. // Pulled out as a pure helper so the bead-scoping logic is unit-testable without diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index f83e66186..eef83fb30 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -498,58 +498,43 @@ func runGateListProxiedServer(cmd *cobra.Command, ctx context.Context, args []st allFlag, _ := cmd.Flags().GetBool("all") limit, _ := cmd.Flags().GetInt("limit") + if len(args) == 0 { + // The workspace-wide listing is the shared entry's, on this route too. + return runGateList(ctx, allFlag, limit) + } + uw, err := proxiedOpenReadUOW(ctx) if err != nil { return err } defer uw.Close(ctx) - if len(args) == 1 { - target, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), args[0]) - if errors.Is(err, storage.ErrNotFound) { - return HandleErrorRespectJSON("issue not found: %s", args[0]) - } - if err != nil { - return HandleErrorRespectJSON("resolving %s: %v", args[0], err) - } - var metas []*types.IssueWithDependencyMetadata - if isWisp { - metas, err = uw.DependencyUseCase().ListWispWithIssueMetadata(ctx, target.ID, domain.DepListFilter{Direction: domain.DepDirectionOut}) - } else { - metas, err = uw.DependencyUseCase().ListWithIssueMetadata(ctx, target.ID, domain.DepListFilter{Direction: domain.DepDirectionOut}) - } - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - deps := make([]*types.Issue, 0, len(metas)) - for _, m := range metas { - if m != nil { - deps = append(deps, &m.Issue) - } - } - gates := filterIssueGates(deps, allFlag, limit) - if jsonOutput { - return outputJSON(gates) - } - displayGates(gates, allFlag) - return nil + target, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), args[0]) + if errors.Is(err, storage.ErrNotFound) { + return HandleErrorRespectJSON("issue not found: %s", args[0]) } - - gateType := types.IssueType("gate") - filter := types.IssueFilter{ - IssueType: &gateType, - Limit: limit, + if err != nil { + return HandleErrorRespectJSON("resolving %s: %v", args[0], err) } - if !allFlag { - filter.ExcludeStatus = []types.Status{types.StatusClosed} + var metas []*types.IssueWithDependencyMetadata + if isWisp { + metas, err = uw.DependencyUseCase().ListWispWithIssueMetadata(ctx, target.ID, domain.DepListFilter{Direction: domain.DepDirectionOut}) + } else { + metas, err = uw.DependencyUseCase().ListWithIssueMetadata(ctx, target.ID, domain.DepListFilter{Direction: domain.DepDirectionOut}) } - page, err := uw.IssueUseCase().SearchIssues(ctx, "", filter) if err != nil { return HandleErrorRespectJSON("%v", err) } - if jsonOutput { - return outputJSON(page.Items) + deps := make([]*types.Issue, 0, len(metas)) + for _, m := range metas { + if m != nil { + deps = append(deps, &m.Issue) + } } - displayGates(page.Items, allFlag) + gates := filterIssueGates(deps, allFlag, limit) + if jsonOutput { + return outputJSON(gates) + } + displayGates(gates, allFlag) return nil } diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 6c46a786b..b4ca8d266 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "dependencyeditor.go", "edgereader.go", "errors.go", + "gates.go", "graphcounter.go", "importer.go", "initverifier.go", @@ -56,6 +57,7 @@ go_test( srcs = [ "claimer_external_test.go", "current_test.go", + "gates_test.go", "issueops_external_test.go", "molecule_internal_test.go", "molecule_test.go", diff --git a/issueops/gates.go b/issueops/gates.go new file mode 100644 index 000000000..fb82460ad --- /dev/null +++ b/issueops/gates.go @@ -0,0 +1,58 @@ +package issueops + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/types" +) + +// GateListExpression is the query `bd gate list` asks: every row of the gate +// type, on both planes, whatever its other fields. It names no status, so the +// Querier's default closed exclusion applies unless GateListRequest.All lifts +// it (QueryRequest.IncludeClosed). +const GateListExpression = "type=gate" + +// GateListRequest describes `bd gate list` over the whole workspace. +type GateListRequest struct { + // All admits closed gates, which the listing otherwise hides. + All bool + // Limit bounds the rows returned; 0 means unlimited. A negative limit is + // the Querier's ErrValidation. + Limit int +} + +// ListGates answers `bd gate list` (no issue argument) through the Querier +// role, which every backend serves (a remote one as queryIssues, which every +// deployed server publishes). It is the one library entry for the listing: +// the CLI's direct, proxied and remote routes each call it with their route's +// Querier, so no route builds a storage filter of its own. +// +// The question is exactly the one the raw filter used to ask: the gate type, +// both planes, closed rows excluded unless All is set, no other default +// suppression (templates, pinned rows and ephemeral gates are all listed), +// in the store's default order, cut at Limit. +// +// A failed read is an error, never an empty list. Rows come back in the +// Querier's order; a row carrying no issue is dropped. +func ListGates(ctx context.Context, querier Querier, req GateListRequest) ([]*types.Issue, error) { + if querier == nil { + return nil, &ErrUnsupported{Op: "ListGates", Backend: "nil"} + } + limit := req.Limit + page, err := querier.Query(ctx, QueryRequest{ + Expression: GateListExpression, + IncludeClosed: req.All, + Limit: &limit, + }) + if err != nil { + return nil, fmt.Errorf("listing gates: %w", err) + } + gates := make([]*types.Issue, 0, len(page.Items)) + for _, row := range page.Items { + if row != nil && row.Issue != nil { + gates = append(gates, row.Issue) + } + } + return gates, nil +} diff --git a/issueops/gates_test.go b/issueops/gates_test.go new file mode 100644 index 000000000..0c2c80274 --- /dev/null +++ b/issueops/gates_test.go @@ -0,0 +1,80 @@ +package issueops_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +func gateQuery(all bool, limit int) issueops.QueryRequest { + return issueops.QueryRequest{Expression: "type=gate", IncludeClosed: all, Limit: &limit} +} + +// TestListGatesAsksTheQuerierTheRawQuestion is S20: `bd gate list` used to +// build a raw SearchIssues filter (gate type, closed excluded unless --all), +// which the http backend refuses. The entry asks the Querier role the same +// question, so every route answers it through a role every server serves. +func TestListGatesAsksTheQuerierTheRawQuestion(t *testing.T) { + cases := []struct { + name string + req issueops.GateListRequest + want issueops.QueryRequest + }{ + {"open gates, default limit", issueops.GateListRequest{Limit: 50}, gateQuery(false, 50)}, + {"unlimited", issueops.GateListRequest{Limit: 0}, gateQuery(false, 0)}, + {"all gates", issueops.GateListRequest{All: true, Limit: 7}, gateQuery(true, 7)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{ids: []string{"g-1", "g-2"}}}} + got, err := issueops.ListGates(context.Background(), q, tc.req) + if err != nil { + t.Fatalf("ListGates: %v", err) + } + if len(q.calls) != 1 || !reflect.DeepEqual(q.calls[0], tc.want) { + t.Fatalf("Query calls = %#v, want one %#v", q.calls, tc.want) + } + ids := make([]string, 0, len(got)) + for _, g := range got { + ids = append(ids, g.ID) + } + if !reflect.DeepEqual(ids, []string{"g-1", "g-2"}) { + t.Fatalf("gates = %v, want [g-1 g-2] in the Querier's order", ids) + } + }) + } +} + +func TestListGatesNeverTakesAFailureForAnEmptyList(t *testing.T) { + refusal := &issueops.ErrUnsupported{Op: "Querier.Query", Backend: "http"} + q := ¤tQueryRecorder{responses: []currentQueryResponse{{err: refusal}}} + got, err := issueops.ListGates(context.Background(), q, issueops.GateListRequest{Limit: 50}) + if !errors.Is(err, refusal) || got != nil { + t.Fatalf("ListGates = (%v, %v), want (nil, the refusal)", got, err) + } +} + +func TestListGatesDropsRowsWithoutAnIssue(t *testing.T) { + q := gateRows{items: []*types.IssueWithCounts{nil, {}, {Issue: &types.Issue{ID: "g-1"}}}} + got, err := issueops.ListGates(context.Background(), q, issueops.GateListRequest{}) + if err != nil || len(got) != 1 || got[0].ID != "g-1" { + t.Fatalf("ListGates = (%v, %v), want ([g-1], nil)", got, err) + } +} + +func TestListGatesRefusesANilQuerier(t *testing.T) { + var unsupported *issueops.ErrUnsupported + if _, err := issueops.ListGates(context.Background(), nil, issueops.GateListRequest{}); !errors.As(err, &unsupported) { + t.Fatalf("ListGates(nil) error = %v, want ErrUnsupported", err) + } +} + +type gateRows struct{ items []*types.IssueWithCounts } + +func (g gateRows) Query(context.Context, issueops.QueryRequest) (issueops.IssuePage, error) { + return issueops.IssuePage{Items: g.items}, nil +} From e6a891b9008cabf87a1133edb90829380116cfe9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 14 Oct 2026 00:18:00 +0000 Subject: [PATCH 03/76] feat(list): bd list --no-assignee, --title and --id over http behind per-parameter tokens (S15b) (#7550) * feat(list): bd list --no-assignee, --title and --id over http behind per-parameter tokens (S15b) Over http `bd list --no-assignee`, `--title` and `--id` refused: the list encoder had no listIssues parameter for ListRequest.NoAssignee, TitleSearch or IDFilter (B3 #11). All three are maintainer-city cutover blockers: gc's default on_boot hook runs `bd list --no-assignee --status=in_progress ...` on every agent boot, the PR-intake formula lists by `--title "$SUBJECT"`, and the pr-review skill by `--id `. The library entry is unchanged: issueops.Reader.List, whose bodies already apply all three through workapi.BuildListFilter, the same entry the local route reads through. - Server: GET /v0/beads/issues decodes `no_assignee`, `title` and `id` into those ListRequest members (`id` handed over as written; the role splits, trims and de-duplicates, as countIssues already does). Each is behind its own new behavior token, issues.list.noAssignee, issues.list.title and issues.list.id (additive, wire_revision stays 2): each is a separate caller's dependency, and a refusal names exactly the one it needed. The spec parameters cite their tokens; apigen and the wire-shape golden are regenerated. - Client: the encoder maps the three members to their parameters, and httpReader.List refuses a request carrying one BEFORE dialing, with ErrUnsupported{Capability: }, unless the handshake advertises that token (refuseUnservedSearch generalised to one table, tokenedListFilters, with bd search's `q`). Never a round trip an older server answers unknown_parameter, never a wider listing. - ErrHTTPUnsupported's text names the capability when it has one, as the portable sentinel does, so `bd list` says which token the server lacks. - The three E-ListRequest ledger rows retire; the ledger is re-rendered. The id scope unparks the served Reader contract: the sixteen cases parked on E-ListRequest.IDFilter now run against the in-process server. Twelve pass as written; four still park, each now on the member it actually needs (Offset twice, PinnedFlag, ExcludeTypes). The served refusal sweep asserts SpecPrefix where it asserted IDFilter. Tests (failing first): handler test that each parameter reaches the storage filter; client tests that each member rides its parameter behind its token and refuses pre-dial, zero dials, naming its token, against a server lacking it; round-trip and vocabulary gates; capability lists. Co-Authored-By: Claude Opus 5.5 * test(backend/http): the four maintainer-city list shapes end to end; corpus marks relabelled SRV-1 (S15b) TestE2E_ListScopeFiltersOverHTTP runs the corpus shapes gc-list-on-boot, gc-list-on-boot-workflow, mc-list-title and mc-list-id through the real bd against the in-process server (which advertises the three new tokens) and requires each to answer an embedded workspace's rows and exactly the rows it names: --no-assignee leaves out an assigned in-progress row on the same lane, --title is case-insensitive, --id lists a closed id under --all. Against the MC production server's handshake shape each shape exits non-zero naming its token and never dials listIssues. The corpus suite masks the served side to reference_server.json, the deployed bd 1.3.0 surface, which advertises none of the tokens, so the four shapes now get the named pre-dial refusal there. Their marks move from S6c+S15b to SRV-1 with that reason, like mc-search. With a scratch profile that adds the three tokens and parameters (not committed) exactly these four shapes report "now passes; remove the pending mark" and nothing else changes. Co-Authored-By: Claude Opus 5.5 * fix(list): adopt bd-enterprise's listIssues `id` contract: repeatable, issues.list.ids (S15b) bd-enterprise already publishes listIssues `id` (E9-S2): a REPEATABLE parameter, one id per value, behind `issues.list.ids`, with its client sending IDFilter split into one value per member. The first cut of S15b took ONE comma-separated `id` behind `issues.list.id`, so the same parameter name had two incompatible meanings: an OSS client's "a,b" is a 400 on an enterprise server, and the enterprise rebase onto main would collide. OSS now publishes enterprise's contract unchanged: - Server: listIDFilter (ported verbatim) decodes the repeated `id` into ListRequest.IDFilter's comma set; more than 200 values, an empty value, a comma, leading or trailing whitespace, a control character or an over-long id is a 400 `invalid_argument` on `id`. The spec parameter is enterprise's (array, maxItems 200); apigen and the wireshape golden are regenerated. - Token: CapListIDs = "issues.list.ids", the enterprise spelling and name on both sides. - Client: the encoder splits the set the way the role does (splitIDSet) and sends one `id` per member; the pre-dial gate fires only when the set names an id. - Tests: list_ids_test.go is enterprise's id cases (the exact set reaches storage, the 200 bound is served, every refusal beside its control, the spec bound matches the constant); the handler, client, round-trip and e2e cases move to the repeated shape and the new token. `no_assignee` and `title` are unchanged. Served tier, http e2e (including TestE2E_ListScopeFiltersOverHTTP) and corpus_served_test pass; with a scratch reference profile advertising the three tokens exactly the four S15b shapes report "now passes". Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 10 + README.md | 5 +- backend/http/BUILD.bazel | 1 + backend/http/corpus/testdata/corpus.yaml | 8 +- backend/http/e2e_list_filters_test.go | 113 ++++++++++ engdocs/design/http-divergence-ledger.md | 5 +- internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 15 +- internal/httpapi/list_ids_test.go | 197 ++++++++++++++++++ internal/httpapi/reads.go | 60 ++++++ internal/httpapi/reads_test.go | 49 +++++ internal/httpapi/routes.go | 35 ++++ internal/httpapi/server_test.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 59 +++++- .../httpapi/wireshape/testdata/golden.json | 26 +++ internal/httpclient/BUILD.bazel | 1 + internal/httpclient/encode/encode.go | 26 +++ internal/httpclient/encode/ledger_fields.go | 5 - internal/httpclient/encode/roundtrip_test.go | 9 +- internal/httpclient/encode/table.go | 15 +- internal/httpclient/errors.go | 7 + internal/httpclient/list_scope_test.go | 100 +++++++++ internal/httpclient/role_reader.go | 66 ++++-- internal/httpclient/roles_test.go | 8 +- internal/httpclient/served_counter_test.go | 6 +- internal/httpclient/served_list_test.go | 17 +- internal/httpclient/served_reader_test.go | 148 ++++--------- internal/httpclient/wire/handshake.go | 17 ++ internal/httpclient/wire/vocabulary_test.go | 13 +- 29 files changed, 861 insertions(+), 164 deletions(-) create mode 100644 backend/http/e2e_list_filters_test.go create mode 100644 internal/httpapi/list_ids_test.go create mode 100644 internal/httpclient/list_scope_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c2b1f1629..2b0209e93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 description and metadata are unchanged, and omitting the flag (or passing `0`) is byte-identical to today's output. JSON output is untouched; `--include-comments` still streams every comment there. +- `GET /v0/beads/issues` accepts `no_assignee`, `title` and a repeatable `id` + (one id per value, at most 200), each behind its own new capability token + (`issues.list.noAssignee`, `issues.list.title`, `issues.list.ids`; additive, + no wire_revision bump; `id` and its token are bd-enterprise's contract, + adopted unchanged), and applies them through + the same Reader role a local `bd list` reads through. Over a connected HTTP + workspace `bd list --no-assignee`, `--title` and `--id` now work instead of + refusing. Against a server that does not advertise a flag's token the + client refuses before dialing, and the refusal names the token (every + capability refusal over HTTP now names its token). - `GET /v0/beads/issues` accepts `q`, `bd search`'s free text, behind the new `issues.list.search` capability token (additive; no wire_revision bump). `issueops.ListRequest` gains `Query`, with `SearchIssues`' meaning diff --git a/README.md b/README.md index 058b949a7..8f16f1a28 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,10 @@ different backend needs `--force`. `bd search`, `bd show --current` and `bd recompute-blocked` work over HTTP too, each in its own way. `bd search` asks the server's listing with its text (`q`, behind the `issues.list.search` capability) and -refuses before dialing against a server that does not advertise it. +refuses before dialing against a server that does not advertise it. `bd list +--no-assignee`, `--title` and `--id` likewise ride the listing's +`no_assignee`, `title` and repeatable `id`, each behind its own capability +(`issues.list.noAssignee`, `issues.list.title`, `issues.list.ids`). `bd show --current` finds the actor's in-progress or hooked issue through the server's boolean query. `bd recompute-blocked` is a no-op that exits 0: the server maintains `is_blocked` on every write, and repairing a stale flag is diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index d96187690..dc812c067 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -32,6 +32,7 @@ go_test( srcs = [ "attach_test.go", "e2e_cli_test.go", + "e2e_list_filters_test.go", "e2e_mol_test.go", "e2e_recompute_blocked_test.go", "e2e_remote_roles_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 532103c1c..7eb95caeb 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -259,11 +259,11 @@ shapes: - id: gc-list-on-boot source: gc workquery.go on_boot recovery (--no-assignee) args: [list, --metadata-field, gc.routed_to=pool-a, --status=in_progress, --no-assignee, --json] - pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + pending: {slice: "SRV-1", reason: "S15b sends --no-assignee as listIssues no_assignee behind issues.list.noAssignee, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListScopeFiltersOverHTTP); remove the mark when reference_server.json is upgraded"} - id: gc-list-on-boot-workflow source: gc workquery.go on_boot recovery, workflow lane args: [list, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --status=in_progress, --no-assignee, --json] - pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + pending: {slice: "SRV-1", reason: "S15b sends --no-assignee as listIssues no_assignee behind issues.list.noAssignee, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListScopeFiltersOverHTTP); remove the mark when reference_server.json is upgraded"} - id: mc-list-meta-status source: mc assets snapshot.md (list --json --limit --metadata-field --status) args: [list, --json, --limit, "50", --metadata-field, gc.routed_to=pool-a, --status, open] @@ -284,11 +284,11 @@ shapes: - id: mc-list-title source: mc mol-normalized-issue-pr-v1 (list --all --include-infra --json --limit --title) args: [list, --all, --include-infra, --json, --limit, "10", --title, "Routed open task"] - pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --title is a text filter the list encoder refuses (B3 #11)"} + pending: {slice: "SRV-1", reason: "S15b sends --title as listIssues title behind issues.list.title, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListScopeFiltersOverHTTP); remove the mark when reference_server.json is upgraded"} - id: mc-list-id source: mc skills/pr-review (list --all --id --json --limit) args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] - pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --id refused by the list encoder (B3 #11)"} + pending: {slice: "SRV-1", reason: "S15b sends --id as listIssues id behind issues.list.ids, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListScopeFiltersOverHTTP); remove the mark when reference_server.json is upgraded"} - id: mc-list-assignee-status source: mc review summaries (list --assignee --json --limit --status) args: [list, --assignee, alice, --json, --limit, "10", --status, open] diff --git a/backend/http/e2e_list_filters_test.go b/backend/http/e2e_list_filters_test.go new file mode 100644 index 000000000..a25398e0e --- /dev/null +++ b/backend/http/e2e_list_filters_test.go @@ -0,0 +1,113 @@ +//go:build cgo + +package bdhttp_test + +import ( + "slices" + "strings" + "testing" +) + +// listScopeShapes are the four maintainer-city `bd list` shapes S15b serves, +// one per corpus shape (backend/http/corpus/testdata/corpus.yaml), with the +// token each needs and the titles each must answer on the scope seed: +// - gc-list-on-boot: gc's default on_boot hook, run on every agent boot; +// - gc-list-on-boot-workflow: the same recovery, workflow lane; +// - mc-list-title: the PR-intake formula's `--title "$SUBJECT"`; +// - mc-list-id: the pr-review skill's `--id `. +var listScopeShapes = []struct { + corpusID string + args []string + token string + want []string +}{ + {"gc-list-on-boot", + []string{"list", "--metadata-field", "gc.routed_to=pool-a", "--status=in_progress", "--no-assignee", "--json"}, + "issues.list.noAssignee", []string{"orphan-ip"}}, + {"gc-list-on-boot-workflow", + []string{"list", "--metadata-field", "gc.run_target=pool-a", "--metadata-field", "gc.kind=workflow", "--status=in_progress", "--no-assignee", "--json"}, + "issues.list.noAssignee", []string{"orphan-wf"}}, + // Case-insensitive substring, every status. + {"mc-list-title", + []string{"list", "--all", "--include-infra", "--json", "--limit", "10", "--title", "CHILD-A"}, + "issues.list.title", []string{"child-a"}}, + // A closed id in the set is listed under --all; ids outside it are not. + {"mc-list-id", + []string{"list", "--all", "--id", "{child-c},{loner}", "--json", "--limit", "0"}, + "issues.list.ids", []string{"child-c", "loner"}}, +} + +// seedListScope adds the on_boot recovery rows to the shared seed: one +// unassigned in-progress row per lane, and an ASSIGNED in-progress row on the +// routed lane, which --no-assignee must leave out. +func (f *remoteRolesFixture) seedListScope() { + f.t.Helper() + ip := f.create("orphan-ip", "-t", "task") + f.mustRun(nil, "update", ip, "-s", "in_progress", "--set-metadata", "gc.routed_to=pool-a") + wf := f.create("orphan-wf", "-t", "task") + f.mustRun(nil, "update", wf, "-s", "in_progress", "--set-metadata", "gc.run_target=pool-a", "--set-metadata", "gc.kind=workflow") + owned := f.create("owned-ip", "-t", "task", "-a", "worker") + f.mustRun(nil, "update", owned, "-s", "in_progress", "--set-metadata", "gc.routed_to=pool-a", "--set-metadata", "gc.run_target=pool-a", "--set-metadata", "gc.kind=workflow") +} + +// TestE2E_ListScopeFiltersOverHTTP is S15b. `bd list --no-assignee`, `--title` +// and `--id` refused over http (the list encoder had no parameter for them); +// they now ride listIssues' `no_assignee`, `title` and `id`, each behind its +// own behavior token, and the server applies them through the same Reader +// role a local `bd list` reads through. Each corpus shape must answer the +// rows an embedded workspace answers, and exactly the rows it names. +// +// Against a server that lacks a token (the MC production server's handshake +// shape, which predates all three) each shape refuses BEFORE dialing +// listIssues, naming the token, rather than answering a wider listing. +func TestE2E_ListScopeFiltersOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + remote := newHTTPFixture(t, bin, addr) + local := newEmbeddedFixture(t, bin) + for _, f := range []*remoteRolesFixture{remote, local} { + f.seed() + f.seedListScope() + } + + for _, shape := range listScopeShapes { + t.Run(shape.corpusID, func(t *testing.T) { + r := remote.run(nil, remote.expand(shape.args)...) + if r.code != 0 { + t.Fatalf("http: bd %s failed (exit %d): stderr=%s", strings.Join(shape.args, " "), r.code, r.stderr) + } + l := local.mustRun(nil, local.expand(shape.args)...) + got, want := remote.titlesOf(r.stdout), local.titlesOf(l.stdout) + if !slices.Equal(got, want) { + t.Errorf("http rows %v, embedded rows %v", got, want) + } + if !slices.Equal(got, shape.want) { + t.Errorf("http rows %v, want exactly %v", got, shape.want) + } + }) + } + + t.Run("MC server mask refuses before dialing", func(t *testing.T) { + proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) + f := newHTTPFixture(t, bin, proxyAddr) + f.ids = remote.ids // same workspace behind the proxy + for _, shape := range listScopeShapes { + counter.reset() + r := f.run(nil, f.expand(shape.args)...) + if r.code == 0 { + t.Errorf("%s: bd %s exited 0 against a server lacking %s: %s", shape.corpusID, strings.Join(shape.args, " "), shape.token, r.stdout) + continue + } + if !strings.Contains(r.stderr, shape.token) { + t.Errorf("%s: the refusal does not name %s: stderr=%s", shape.corpusID, shape.token, r.stderr) + } + for _, p := range counter.snapshot() { + if p == "GET /v0/beads/issues" { + t.Errorf("%s: dialed listIssues against a server lacking %s: %v", shape.corpusID, shape.token, counter.snapshot()) + break + } + } + } + }) +} diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index a3e9034e3..01d95c9a3 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -264 rows: 204 refuse, 34 degrade, 26 retired. +261 rows: 201 refuse, 34 degrade, 26 retired. ## Whole-behavior divergences (D9 ledger) @@ -178,9 +178,7 @@ The listing publishes far more filters than the v0 wire does; every one the wire | ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | |---|---|---|---|---|---|---| -| E-ListRequest.TitleSearch | refuse | `ListRequest.TitleSearch` | a title search refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.SpecPrefix | refuse | `ListRequest.SpecPrefix` | a spec-id prefix restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.IDFilter | refuse | `ListRequest.IDFilter` | an explicit id set on the LISTING refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml); the exact-ids question is getIssue's, reached through the SearchIssues bridge's exact-ids shape (D11), not through a listing filter | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.LabelPattern | refuse | `ListRequest.LabelPattern` | a glob label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.LabelRegex | refuse | `ListRequest.LabelRegex` | a regex label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — as for LabelPattern, listReadyWork publishes label_regex and the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.TitleContains | refuse | `ListRequest.TitleContains` | a title substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | @@ -197,7 +195,6 @@ The listing publishes far more filters than the v0 wire does; every one the wire | E-ListRequest.DueAfter | refuse | `ListRequest.DueAfter` | a due-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.DueBefore | refuse | `ListRequest.DueBefore` | a due-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.EmptyDesc | refuse | `ListRequest.EmptyDesc` | the empty-description predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.NoAssignee | refuse | `ListRequest.NoAssignee` | the unassigned predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes `unassigned`; the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.NoLabels | refuse | `ListRequest.NoLabels` | the no-labels predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.Priority | refuse | `ListRequest.Priority` | an exact priority filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes `priority`; the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.PriorityMin | refuse | `ListRequest.PriorityMin` | a minimum-priority bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 4d50cf5c1..efe2f12b3 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -91,6 +91,7 @@ go_test( "get_issue_test.go", "large_batch_apply_test.go", "list_comments_omitted_test.go", + "list_ids_test.go", "list_sort_test.go", "memories_test.go", "metadata_cas_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 4183dd53b..7a65483eb 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1101,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, and `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, and `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2417,6 +2417,19 @@ type ListIssuesParams struct { // Behind the `issues.list.search` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses a search locally against a server that does not advertise the token, rather than paying for that round trip. Q *string `form:"q,omitempty" json:"q,omitempty"` + // NoAssignee Only issues with no assignee: `bd list --no-assignee`, handed to the role as written beside the other filters. + // + // Behind the `issues.list.noAssignee` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or listing assigned issues as well. + NoAssignee *bool `form:"no_assignee,omitempty" json:"no_assignee,omitempty"` + + // Title Case-insensitive substring match on the title: `bd list --title`. It narrows the listing and lifts none of its default exclusions. + // + // Behind the `issues.list.title` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering the unnarrowed listing. + Title *string `form:"title,omitempty" json:"title,omitempty"` + + // Id Restrict to these canonical ids (repeat the parameter). AND-ed with every other filter, including the default exclusions, so a caller addressing rows regardless of status, type or tier also sends all=true, include_templates, include_gates, include_infra and include_ephemeral. At most 200 values; an id longer than storage holds, an empty one, one with leading or trailing whitespace, or one carrying a control character or a comma is a 400 `invalid_argument` on `param: "id"` — each value is matched exactly as sent, never trimmed or split. An id that names no row is simply not matched. Advertised by the behavior token `issues.list.ids`. + Id *[]string `form:"id,omitempty" json:"id,omitempty"` + // Sort Which of the two served total orders this page is in. `created` is `(created_at DESC, id ASC)`; `priority` is `(priority ASC, created_at DESC, id ASC)` — `bd list`'s flagless ordering, and the order `bd list --sort priority` produces. // // THE DEFAULT IS `created` AND WILL NOT CHANGE. It is what this operation served before this parameter existed, so moving it would alter which rows a truncated page contains for every client written against that behavior, with no error to notice it by. diff --git a/internal/httpapi/list_ids_test.go b/internal/httpapi/list_ids_test.go new file mode 100644 index 000000000..91184e063 --- /dev/null +++ b/internal/httpapi/list_ids_test.go @@ -0,0 +1,197 @@ +package httpapi + +import ( + "net/http" + "net/url" + "reflect" + "slices" + "strconv" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// listIssues' repeatable `id` (IDFilter, behind issues.list.ids). The member +// was already on issueops.ListRequest; what these pin is the TRANSPORT — the +// repeated parameter reaching the role as the comma set it splits, and the +// bounds the document states being enforced before a request reaches storage. +// The contract and these cases are bd-enterprise's (E9-S2), adopted unchanged +// (its copy of this file also covers include_comments, which OSS does not +// publish). + +// TestListIDsReachStorageAsTheExactSet pins the decode: repeated `id` values +// arrive on the storage filter as exactly those ids, and nothing else about the +// request moves. The default exclusions stay in place (the parameter is AND-ed +// with them), which is why the import pre-read sends the five include flags. +func TestListIDsReachStorageAsTheExactSet(t *testing.T) { + filter := listSortOf(t, "?id=bd-2&id=bd-1&id=bd-wisp-9") + if want := []string{"bd-2", "bd-1", "bd-wisp-9"}; !slices.Equal(filter.IDs, want) { + t.Errorf("filter.IDs = %v, want %v", filter.IDs, want) + } + if len(filter.ExcludeStatus) == 0 { + t.Error("an id scope dropped the default status exclusions; the document says `id` is AND-ed with them") + } + + // The control: no `id` is no id scope at all, not an empty one. + if got := listSortOf(t, "").IDs; len(got) != 0 { + t.Errorf("a request with no `id` reached storage with IDs = %v, want none", got) + } + + // Every exclusion off: the shape the client's bulk reads send. The filter + // keeps the id set and NOTHING ELSE that could narrow it — not merely the + // suppressions known today, which is why the check is every field of the + // filter rather than a list of them: a suppression added to the listing + // tomorrow that none of the five flags lifts has to fail here, or the + // import pre-read misses that row kind without a sound. + all := listSortOf(t, "?id=bd-1&all=true&include_templates=true&include_gates=true&include_infra=true&include_ephemeral=true") + if !slices.Equal(all.IDs, []string{"bd-1"}) { + t.Errorf("filter.IDs = %v, want [bd-1]", all.IDs) + } + if extra := setFilterFields(all, "IDs", "Limit", "SortBy"); len(extra) != 0 { + t.Errorf("the every-row-kind request still narrows the id set: %s set on the storage filter %+v", strings.Join(extra, ", "), all) + } +} + +// setFilterFields names every field of a storage filter that is set — a +// non-empty slice or map, a non-nil pointer, a true bool, a non-empty string, a +// non-zero number — other than the ones skipped. +func setFilterFields(f types.IssueFilter, skip ...string) []string { + v := reflect.ValueOf(f) + var set []string + for i := range v.NumField() { + name := v.Type().Field(i).Name + if slices.Contains(skip, name) { + continue + } + field := v.Field(i) + switch field.Kind() { + case reflect.Slice, reflect.Map: + if field.Len() > 0 { + set = append(set, name) + } + default: + if !field.IsZero() { + set = append(set, name) + } + } + } + return set +} + +// TestListIDsAtTheBoundIsServed is the inclusive edge of maxItems: exactly 200 +// ids is a request the document admits. +func TestListIDsAtTheBoundIsServed(t *testing.T) { + filter := listSortOf(t, "?"+idQuery(maxListIDs)) + if len(filter.IDs) != maxListIDs { + t.Errorf("len(filter.IDs) = %d, want %d", len(filter.IDs), maxListIDs) + } +} + +func idQuery(n int) string { + v := url.Values{} + for i := range n { + v.Add("id", "bd-"+strconv.Itoa(i+1)) + } + return v.Encode() +} + +// TestListIDsRefusals pins every refusal the member +// carries, each beside a control that differs in exactly the refused value, so a +// 400 cannot be read as some other parameter's. +func TestListIDsRefusals(t *testing.T) { + long := strings.Repeat("x", types.MaxFieldLen+1) + for _, tc := range []struct { + name string + path string + control string + param string + }{ + { + name: "more ids than maxItems", + path: "/v0/beads/issues?" + idQuery(maxListIDs+1), + control: "/v0/beads/issues?" + idQuery(maxListIDs), + param: "id", + }, + { + // The role splits the joined set on commas, so a value carrying + // one would silently address ids the caller never named. + name: "an id carrying a comma", + path: "/v0/beads/issues?id=" + url.QueryEscape("bd-1,bd-2"), + control: "/v0/beads/issues?id=bd-1&id=bd-2", + param: "id", + }, + { + name: "an id carrying a control character", + path: "/v0/beads/issues?id=" + url.QueryEscape("bd-1\n"), + control: "/v0/beads/issues?id=bd-1", + param: "id", + }, + { + name: "an empty id", + path: "/v0/beads/issues?id=bd-1&id=", + control: "/v0/beads/issues?id=bd-1", + param: "id", + }, + { + // The role trims every member of the set, so a padded value + // would silently address the trimmed id — a row the caller never + // named, while the padded one (which storage can hold) is missed. + name: "an id with leading whitespace", + path: "/v0/beads/issues?id=" + url.QueryEscape(" bd-1"), + control: "/v0/beads/issues?id=bd-1", + param: "id", + }, + { + // Unicode whitespace too: the rule is strings.TrimSpace's, the + // role's own trim, not an ASCII-space check. + name: "an id with trailing non-breaking space", + path: "/v0/beads/issues?id=" + url.QueryEscape("bd-1\u00a0"), + control: "/v0/beads/issues?id=bd-1", + param: "id", + }, + { + name: "an id longer than storage holds", + path: "/v0/beads/issues?id=" + long, + control: "/v0/beads/issues?id=" + long[:types.MaxFieldLen], + param: "id", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + + resp := ts.get(t, tc.path) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("GET %s: status = %d, want 400", tc.path, resp.StatusCode) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) || body["param"] != tc.param || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("body = %v, want invalid_argument/invalid_value on param %s", body, tc.param) + } + if n := len(rec.searchFilters()); n != 0 { + t.Errorf("%d list queries ran; a refused request must not reach storage", n) + } + if got := ts.get(t, tc.control); got.StatusCode != http.StatusOK { + t.Fatalf("control GET %s: status = %d, want 200; body=%v", tc.control, got.StatusCode, decodeBody(t, got)) + } + }) + } +} + +// TestListIDsBoundMatchesTheSpec holds the document's number to the constant +// the handler enforces, so the two cannot drift: a spec that said 300 would be +// refused by the server it documents at a number no reader predicted. +func TestListIDsBoundMatchesTheSpec(t *testing.T) { + doc := loadSpec(t) + so, ok := specOps(t, doc)[OpListIssues] + if !ok { + t.Fatalf("operation %q missing from the spec", OpListIssues) + } + schema := mapAt(t, specParam(t, so, "id"), "schema") + if got, ok := schema["maxItems"].(int); !ok || got != maxListIDs { + t.Errorf("listIssues id maxItems = %v, want %d", schema["maxItems"], maxListIDs) + } + if !slices.Contains(Capabilities(), CapListIDs) { + t.Errorf("Capabilities() does not advertise %s", CapListIDs) + } +} diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 2e616f314..234d1fe18 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -1,6 +1,7 @@ package httpapi import ( + "fmt" "net/http" "strings" @@ -355,6 +356,9 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { Status: q.csv("status"), IssueType: q.str("type"), Assignee: q.str("assignee"), + // `bd list --no-assignee`, behind issues.list.noAssignee + // (CapIssuesListNoAssignee). + NoAssignee: q.boolean("no_assignee"), Labels: q.list("label"), LabelsAny: q.list("label_any"), @@ -362,6 +366,14 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { ParentID: q.str("parent"), + // `bd list --title`, behind issues.list.title (CapIssuesListTitle). + TitleSearch: q.str("title"), + // `bd list --id`, behind issues.list.ids (CapListIDs). The id scope + // arrives as a repeated parameter and reaches the role as the + // comma-separated set issueops.ListRequest.IDFilter is. The join is + // lossless only because listIDFilter refuses a comma inside a value. + IDFilter: listIDFilter(q), + Brief: q.boolean("brief"), AllFlag: q.boolean("all"), IncludeTemplates: q.boolean("include_templates"), @@ -449,6 +461,54 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { writeJSON(w, body) } +// maxListIDs bounds listIssues' repeatable `id` filter. It is the spec's +// `maxItems`, and TestListIDsBoundMatchesTheSpec holds the two together. +const maxListIDs = 200 + +// listIDFilter decodes listIssues' repeatable `id` filter into the comma-separated +// set issueops.ListRequest.IDFilter takes. The contract is bd-enterprise's +// (E9-S2), adopted unchanged. +// +// Every value is bounded the way a path id is (customMethodTarget): longer than +// the column holds, or carrying a control character, names no row that can +// exist. Unlike the path, a malformed value here is a 400 naming `id` rather +// than a 404 — there is no single resource to be missing — and two more shapes +// are refused because of what the role does with the joined set: a COMMA, +// because the role splits the set on commas and a value carrying one would +// silently address two ids the caller never named; and LEADING OR TRAILING +// WHITESPACE, because the role trims every member, so a padded value would +// silently address the trimmed id — a different one from the id named. +func listIDFilter(q *query) string { + vals := q.list("id") + if len(vals) == 0 { + return "" + } + if len(vals) > maxListIDs { + q.invalid("id", fmt.Sprintf("at most %d id values per request; got %d", maxListIDs, len(vals))) + return "" + } + for _, id := range vals { + switch { + case strings.TrimSpace(id) == "": + q.invalid("id", "an id value is empty") + return "" + case strings.Contains(id, ","): + q.invalid("id", fmt.Sprintf("%q carries a comma; repeat the parameter once per id instead", id)) + return "" + case strings.ContainsFunc(id, isControlChar): + q.invalid("id", "an id value carries a control character") + return "" + case strings.TrimSpace(id) != id: + q.invalid("id", fmt.Sprintf("%q has leading or trailing whitespace; send the canonical id", id)) + return "" + case types.CheckFieldLen("id", id) != nil: + q.invalid("id", fmt.Sprintf("an id value is longer than %d characters", types.MaxFieldLen)) + return "" + } + } + return strings.Join(vals, ",") +} + // handleQueryIssues answers GET /v0/beads/issues:query. // // The EXPRESSION IS NOT PARSED HERE. Parsing, evaluation, the predicate diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index f67dd3fe5..604fccadc 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -3,6 +3,7 @@ package httpapi import ( "context" "net/http" + "slices" "strings" "sync" "testing" @@ -500,3 +501,51 @@ func TestTheEphemeralPlaneParameterIsTheSkewSignal(t *testing.T) { t.Errorf("body = %v, want param=include_ephemerals reason=unknown_parameter", body) } } + +// TestListForwardsTheScopeFilters is the handler half of S15b: listIssues' +// `no_assignee` (issues.list.noAssignee), `title` (issues.list.title) and `id` +// (issues.list.ids, repeatable) reach the storage seam through the Reader role as the +// ListRequest members `bd list --no-assignee`, `--title` and `--id` set, and +// their absence narrows nothing. +func TestListForwardsTheScopeFilters(t *testing.T) { + for _, tc := range []struct { + name, path string + check func(t *testing.T, f types.IssueFilter) + }{ + {"absent narrows nothing", "/v0/beads/issues", func(t *testing.T, f types.IssueFilter) { + if f.NoAssignee || f.TitleSearch != "" || len(f.IDs) != 0 { + t.Errorf("filter = {NoAssignee:%v TitleSearch:%q IDs:%v}, want none set", f.NoAssignee, f.TitleSearch, f.IDs) + } + }}, + {"no_assignee", "/v0/beads/issues?no_assignee=true&status=in_progress", func(t *testing.T, f types.IssueFilter) { + if !f.NoAssignee { + t.Error("no_assignee=true did not reach the filter") + } + }}, + {"title", "/v0/beads/issues?title=Routed+open+task", func(t *testing.T, f types.IssueFilter) { + if f.TitleSearch != "Routed open task" { + t.Errorf("TitleSearch = %q, want %q", f.TitleSearch, "Routed open task") + } + if f.TitleContains != "" { + t.Errorf("title leaked into TitleContains = %q", f.TitleContains) + } + }}, + {"id is repeatable, one id per value", "/v0/beads/issues?id=e2e-r1&id=e2e-a1&all=true", func(t *testing.T, f types.IssueFilter) { + if !slices.Equal(f.IDs, []string{"e2e-r1", "e2e-a1"}) { + t.Errorf("IDs = %v, want [e2e-r1 e2e-a1]", f.IDs) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + if resp := ts.get(t, tc.path); resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d, want 200", tc.path, resp.StatusCode) + } + filters := rec.searchFilters() + if len(filters) != 1 { + t.Fatalf("search calls = %d, want 1", len(filters)) + } + tc.check(t, filters[0]) + }) + } +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 206a798c8..6789f992a 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -132,6 +132,40 @@ const CapIssuesCountScope = "issues.count.scope" // is never widened into an unsearched listing. const CapIssuesListSearch = "issues.list.search" +// CapIssuesListNoAssignee and CapIssuesListTitle are the behavior +// capabilities that advertise GET /v0/beads/issues' `no_assignee` and `title` +// parameters (reads.go, spec OpListIssues): `bd list --no-assignee` and +// `--title`, carried as issueops.ListRequest's NoAssignee and TitleSearch and +// applied by the same Reader role (workapi.BuildListFilter) the local route +// reads through. Like CapIssuesListSearch each names a PARAMETER added to an +// existing operation, so each rides behaviorCapabilities and is the token its +// parameter's OpenAPI description cites for +// TestNewParameterOnExistingOperationHasABehaviorToken. +// +// One token per parameter rather than one for the set: each is a separate +// cutover dependency of a separate caller (gc's on_boot recovery, the PR-intake +// formula; CapListIDs below is the pr-review skill's), and a client names +// exactly the one a refused request needed. +// +// The client (internal/httpclient) refuses a request carrying either locally +// against a server that does not advertise its token, with ErrUnsupported +// naming the token: an older server answers the parameter with +// `unknown_parameter`, and dropping it would answer a WIDER listing than asked +// for (for gc's on_boot recovery, every in-progress bead, assigned or not). +const ( + CapIssuesListNoAssignee = "issues.list.noAssignee" + CapIssuesListTitle = "issues.list.title" +) + +// CapListIDs is the behavior capability that advertises the repeatable `id` +// filter on listIssues (issueops.ListRequest.IDFilter; `bd list --id`). It +// rides beside `issues.list` for the reason CapIssuesListSort does: the +// operation already exists, and what this announces is one more member it +// understands. The contract — one id per value, at most maxListIDs values, +// each matched exactly as sent — and the token's spelling are bd-enterprise's +// (E9-S2), adopted unchanged so the two servers publish one `id`. +const CapListIDs = "issues.list.ids" + // CapIssuesSweepWispsPlane is the behavior capability that advertises the // `wisps-plane` value of SweepRequest.tier (sweep.go, spec SweepRequest.tier): // a third, wider tier that clears the whole wisps table plus any durable bead @@ -1000,6 +1034,7 @@ var behaviorCapabilities = []string{ CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapIssuesCloseAutoCloseMolecule, CapIssuesListSearch, + CapIssuesListNoAssignee, CapIssuesListTitle, CapListIDs, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 90af30763..d4a6b27f0 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -638,7 +638,8 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", - "issues.close.autoCloseMolecule", "issues.list.search") + "issues.close.autoCloseMolecule", "issues.list.search", + "issues.list.noAssignee", "issues.list.title", "issues.list.ids") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index e9db266a3..de7f04b33 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1318,6 +1318,55 @@ paths: that round trip. schema: type: string + - name: no_assignee + in: query + description: >- + Only issues with no assignee: `bd list --no-assignee`, handed to + the role as written beside the other filters. + + + Behind the `issues.list.noAssignee` behavior token: an older server + answers `unknown_parameter` for this name. A client with the + handshake's capabilities cached refuses the request locally against + a server that does not advertise the token, rather than paying for + that round trip or listing assigned issues as well. + schema: + type: boolean + default: false + - name: title + in: query + description: >- + Case-insensitive substring match on the title: `bd list --title`. + It narrows the listing and lifts none of its default exclusions. + + + Behind the `issues.list.title` behavior token: an older server + answers `unknown_parameter` for this name. A client with the + handshake's capabilities cached refuses the request locally against + a server that does not advertise the token, rather than paying for + that round trip or answering the unnarrowed listing. + schema: + type: string + - name: id + in: query + description: >- + Restrict to these canonical ids (repeat the parameter). AND-ed with + every other filter, including the default exclusions, so a caller + addressing rows regardless of status, type or tier also sends + all=true, include_templates, include_gates, include_infra and + include_ephemeral. At most 200 values; an id longer than storage + holds, an empty one, one with leading or trailing whitespace, or one + carrying a control character or a comma is a 400 `invalid_argument` + on `param: "id"` — each value is matched exactly as sent, never + trimmed or split. An id that names no row is simply not matched. + Advertised by the behavior token `issues.list.ids`. + style: form + explode: true + schema: + type: array + maxItems: 200 + items: + type: string - name: sort in: query description: >- @@ -8247,10 +8296,16 @@ components: `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the - member unknown, and `issues.list.search`, which announces that + member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering - `unknown_parameter`. One behavior token is + `unknown_parameter`, and `issues.list.noAssignee`, + `issues.list.title` and `issues.list.ids`, which announce that + `GET /v0/beads/issues` accepts `no_assignee`, `title` and the + repeatable `id` respectively (`bd list --no-assignee`, `--title` + and `--id`; see those parameters) rather than answering + `unknown_parameter`. One + behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 29d527ff6..14fb03288 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -5775,6 +5775,15 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:id", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:include_ephemeral", @@ -5847,6 +5856,15 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:no_assignee", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, { "schema": "param:listIssues", "member": "query:parent", @@ -5885,6 +5903,14 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:title", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:type", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 2dbd3ee07..611b53d50 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -117,6 +117,7 @@ go_test( "insecure_credential_guard_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", + "list_scope_test.go", "list_search_test.go", "list_sort_test.go", "metadatacas_test.go", diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 3db3f19bd..e580d79b8 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -35,6 +35,7 @@ import ( "reflect" "sort" "strconv" + "strings" "time" "github.com/steveyegge/beads/internal/types" @@ -289,6 +290,7 @@ func ListParams(req issueops.ListRequest) (url.Values, error) { b.str("status", req.Status) b.str("type", req.IssueType) b.str("assignee", req.Assignee) + b.boolean("no_assignee", req.NoAssignee) b.list("label", req.Labels) b.list("label_any", req.LabelsAny) @@ -296,6 +298,8 @@ func ListParams(req issueops.ListRequest) (url.Values, error) { b.str("parent", req.ParentID) b.str("q", req.Query) + b.str("title", req.TitleSearch) + b.list("id", splitIDSet(req.IDFilter)) b.boolean("all", req.AllFlag) b.boolean("include_templates", req.IncludeTemplates) @@ -314,6 +318,28 @@ func ListParams(req issueops.ListRequest) (url.Values, error) { return b.done() } +// splitIDSet reads ListRequest.IDFilter's comma-separated set the way the role +// does (workapi.BuildListFilter: split on commas, trim, drop empties and +// repeats) and hands back one member per listIssues `id` value, which is the +// repeatable shape the server takes (issues.list.ids). A set naming no id +// sends no `id` at all. +func splitIDSet(set string) []string { + if set == "" { + return nil + } + var out []string + seen := map[string]bool{} + for _, id := range strings.Split(set, ",") { + id = strings.TrimSpace(id) + if id == "" || seen[id] { + continue + } + seen[id] = true + out = append(out, id) + } + return out +} + // QueryParams encodes a boolean-expression query onto // GET /v0/beads/issues:query. func QueryParams(req issueops.QueryRequest) (url.Values, error) { diff --git a/internal/httpclient/encode/ledger_fields.go b/internal/httpclient/encode/ledger_fields.go index 30aae447b..18709c6fc 100644 --- a/internal/httpclient/encode/ledger_fields.go +++ b/internal/httpclient/encode/ledger_fields.go @@ -108,10 +108,7 @@ func listRequestRows() []Row { return refusal("E-ListRequest."+field, tyListRequest, field, what, why, fieldSpec) } rows := []Row{ - listRefusal("TitleSearch", "a title search refuses", noWireList), listRefusal("SpecPrefix", "a spec-id prefix restriction refuses", noWireList), - listRefusal("IDFilter", "an explicit id set on the LISTING refuses", - noWireList+"; the exact-ids question is getIssue's, reached through the SearchIssues bridge's exact-ids shape (D11), not through a listing filter"), listRefusal("LabelPattern", "a glob label filter refuses", noWireList+" — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through"), listRefusal("LabelRegex", "a regex label filter refuses", @@ -130,8 +127,6 @@ func listRequestRows() []Row { listRefusal("DueAfter", "a due-after bound refuses", noWireList), listRefusal("DueBefore", "a due-before bound refuses", noWireList), listRefusal("EmptyDesc", "the empty-description predicate refuses", noWireList), - listRefusal("NoAssignee", "the unassigned predicate refuses", - noWireList+" — listReadyWork publishes `unassigned`; the listing does not"), listRefusal("NoLabels", "the no-labels predicate refuses", noWireList), listRefusal("Priority", "an exact priority filter refuses", noWireList+" — listReadyWork publishes `priority`; the listing does not"), diff --git a/internal/httpclient/encode/roundtrip_test.go b/internal/httpclient/encode/roundtrip_test.go index 6e2971693..f03f89604 100644 --- a/internal/httpclient/encode/roundtrip_test.go +++ b/internal/httpclient/encode/roundtrip_test.go @@ -176,13 +176,18 @@ func roundTripCases() []roundTripCase { Status: "open,in_progress", IssueType: "bug", Assignee: "agent-7", + // Beside Assignee on purpose: the wire carries both as written and the + // role decides what the pair means. + NoAssignee: true, Labels: []string{"alpha", "beta"}, LabelsAny: []string{"gamma"}, ExcludeLabels: []string{"delta", "epsilon"}, - ParentID: "bd-epic1", - Query: "login bug", + ParentID: "bd-epic1", + Query: "login bug", + TitleSearch: "Routed open task", + IDFilter: "bd-1,bd-wisp-9,bd-42", AllFlag: true, IncludeTemplates: true, diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index 462598658..b24c44365 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -326,12 +326,19 @@ func listTable() Table { param("Status", "status"), paramTo("IssueType", "type", "IssueType"), param("Assignee", "assignee"), - refused("TitleSearch", "E-ListRequest.TitleSearch"), + // TitleSearch, IDFilter and NoAssignee (S15b) are each behind their own + // behavior token — issues.list.title, issues.list.ids, + // issues.list.noAssignee: httpReader.List refuses a request carrying + // one, before dialing, unless the handshake advertises its token + // (role_reader.go refuseUnservedListFilters). IDFilter is split the + // way the role splits it and sent as the repeatable `id`, one value + // per member (bd-enterprise's contract, adopted unchanged). + paramTo("TitleSearch", "title", "TitleSearch"), refused("SpecPrefix", "E-ListRequest.SpecPrefix"), - refused("IDFilter", "E-ListRequest.IDFilter"), + paramTo("IDFilter", "id", "IDFilter"), // Behind issues.list.search: httpReader.List refuses a request // carrying it, before dialing, unless the handshake advertises the - // token (role_reader.go refuseUnservedSearch). + // token (role_reader.go refuseUnservedListFilters). param("Query", "q"), param("Labels", "label"), param("LabelsAny", "label_any"), @@ -354,7 +361,7 @@ func listTable() Table { refused("DueAfter", "E-ListRequest.DueAfter"), refused("DueBefore", "E-ListRequest.DueBefore"), refused("EmptyDesc", "E-ListRequest.EmptyDesc"), - refused("NoAssignee", "E-ListRequest.NoAssignee"), + paramTo("NoAssignee", "no_assignee", "NoAssignee"), refused("NoLabels", "E-ListRequest.NoLabels"), dropped("SkipLabels", "E-ListRequest.SkipLabels", "the wire hydrates labels either way; dropping the opt-out costs a join and hands the caller more data, never less"), diff --git a/internal/httpclient/errors.go b/internal/httpclient/errors.go index a4a764acc..23a424b14 100644 --- a/internal/httpclient/errors.go +++ b/internal/httpclient/errors.go @@ -39,6 +39,13 @@ func (e *ErrHTTPUnsupported) Error() string { if e.ServerURL == "" { return e.Unsup.Error() } + // A capability refusal names its token, as the portable sentinel does: + // "Reader.List" alone cannot tell a caller which of a listing's tokened + // filters the server lacks. + if e.Unsup.Capability != "" { + return fmt.Sprintf("operation %q not supported by the http backend (capability %q not advertised by bd serve at %s)", + e.Unsup.Op, e.Unsup.Capability, e.ServerURL) + } return fmt.Sprintf("operation %q not supported by the http backend (bd serve at %s)", e.Unsup.Op, e.ServerURL) } diff --git a/internal/httpclient/list_scope_test.go b/internal/httpclient/list_scope_test.go new file mode 100644 index 000000000..c4f67ce07 --- /dev/null +++ b/internal/httpclient/list_scope_test.go @@ -0,0 +1,100 @@ +package httpclient + +import ( + "errors" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// listScopeCases are S15b's three ListRequest members, each with the +// listIssues parameter it rides and the behavior token that gates it: +// `bd list --no-assignee` (gc's on_boot recovery), `--title` (the PR-intake +// formula) and `--id` (the pr-review skill). +var listScopeCases = []struct { + name, param, token string + set func(*issueops.ListRequest) + want []string +}{ + {"NoAssignee", "no_assignee", wire.CapListNoAssignee, + func(r *issueops.ListRequest) { r.NoAssignee = true }, []string{"true"}}, + {"TitleSearch", "title", wire.CapListTitle, + func(r *issueops.ListRequest) { r.TitleSearch = "Routed open task" }, []string{"Routed open task"}}, + {"IDFilter", "id", wire.CapListIDs, + // Split the way the role splits it, one `id` value per member. + func(r *issueops.ListRequest) { r.IDFilter = "e2e-r1, e2e-a1,e2e-r1" }, []string{"e2e-r1", "e2e-a1"}}, +} + +// TestListScopeFiltersRideTheirParametersBehindTheirTokens is S15b's client +// half: each member reaches listIssues as its own parameter once the +// handshake advertises its token. +func TestListScopeFiltersRideTheirParametersBehindTheirTokens(t *testing.T) { + for _, tc := range listScopeCases { + t.Run(tc.name, func(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{"issues.list", tc.token}} + w := newCountingWire(`{"items":[],"has_more":false}`) + limit := 10 + req := issueops.ListRequest{Limit: &limit} + tc.set(&req) + if _, err := listRoleWithSnapshot(t, w, served).List(t.Context(), req); err != nil { + t.Fatalf("List: %v", err) + } + if got := w.query(t)[tc.param]; !slices.Equal(got, tc.want) { + t.Errorf("query[%s] = %v, want %v (full query: %v)", tc.param, got, tc.want, w.query(t)) + } + }) + } +} + +// TestListScopeFiltersRefuseLocallyWhenTheServerLacksTheToken is the skew +// side: against a server that does not advertise a member's token (the +// deployed reference server, for one), a request carrying it refuses BEFORE +// dialing with the typed capability error naming that token. Sending it would +// cost a guaranteed 400 unknown_parameter; dropping it would answer a wider +// listing than asked for — for gc's on_boot recovery, every in-progress bead, +// assigned or not. +func TestListScopeFiltersRefuseLocallyWhenTheServerLacksTheToken(t *testing.T) { + for _, tc := range listScopeCases { + t.Run(tc.name, func(t *testing.T) { + // Every token but this one: each member is gated by its own. + var caps []string + for _, other := range listScopeCases { + if other.token != tc.token { + caps = append(caps, other.token) + } + } + masked := &apigen.ContextResponse{Capabilities: append([]string{"issues.list", wire.CapListSort}, caps...)} + w := newCountingWire(`{"items":[],"has_more":false}`) + req := issueops.ListRequest{} + tc.set(&req) + _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), req) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("List error = %v, want *storage.ErrUnsupported", err) + } + if unsup.Capability != tc.token || unsup.Op != "Reader.List" { + t.Errorf("refusal = {Op: %q, Capability: %q}, want {Reader.List, %s}", unsup.Op, unsup.Capability, tc.token) + } + if !strings.Contains(err.Error(), tc.token) { + t.Errorf("refusal text %q does not name %s; a user cannot tell which filter the server lacks", err, tc.token) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.dialed)) + } + + // A listing without the member is unaffected by the missing token. + limit := 10 + if _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), issueops.ListRequest{Limit: &limit}); err != nil { + t.Fatalf("List without %s against the same server: %v", tc.name, err) + } + if q := w.query(t); q.Has(tc.param) { + t.Errorf("a listing without %s sent %s: %v", tc.name, tc.param, q) + } + }) + } +} diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index 77eef9a2b..9c38282f7 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -6,6 +6,7 @@ import ( "context" "net/http" "slices" + "strings" "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/encode" @@ -63,7 +64,7 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue // List serves listIssues, through the pager in list_walk.go. func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { - if err := r.refuseUnservedSearch(ctx, req); err != nil { + if err := r.refuseUnservedListFilters(ctx, req); err != nil { return issueops.IssuePage{}, err } params, err := encode.ListParams(req) @@ -73,28 +74,53 @@ func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueop return r.store.walkIssues(ctx, params, req) } -// refuseUnservedSearch is the pre-dial gate for ListRequest.Query, which rides -// listIssues' `q` only on a server advertising issues.list.search. An older -// server answers `q` with 400 unknown_parameter, so a search against it is -// refused HERE, with the typed capability error naming the token: never a -// round trip that is refused anyway, and never the unsearched listing passed -// off as the search. +// tokenedListFilters are the ListRequest members listIssues carries only on a +// server advertising a behavior token of their own, each with that token: +// `bd search`'s text (S26) and `bd list --title`, `--id` and `--no-assignee` +// (S15b). Order decides which token a request carrying several names; each +// refusal is complete on its own, so any order is honest. +var tokenedListFilters = []struct { + token string + set func(issueops.ListRequest) bool +}{ + {wire.CapListSearch, func(r issueops.ListRequest) bool { return r.Query != "" }}, + {wire.CapListNoAssignee, func(r issueops.ListRequest) bool { return r.NoAssignee }}, + {wire.CapListTitle, func(r issueops.ListRequest) bool { return r.TitleSearch != "" }}, + // An id set naming no id (only commas and blanks) sends no `id`, so it + // needs no token. + {wire.CapListIDs, func(r issueops.ListRequest) bool { + return strings.TrimSpace(strings.ReplaceAll(r.IDFilter, ",", "")) != "" + }}, +} + +// refuseUnservedListFilters is the pre-dial gate for the tokened members. An +// older server answers each one's parameter with 400 unknown_parameter, so a +// request carrying one against a server that does not advertise its token is +// refused HERE, with the typed capability error naming that token: never a +// round trip that is refused anyway, and never a wider listing (the +// unsearched one, every in-progress bead instead of the unassigned ones, the +// whole workspace instead of an id set) passed off as the answer. // // listIssues is a baseline operation, so dispatch preflights nothing for it; -// this is the one handshake a search costs, and a request without Query never -// asks for it. -func (r httpReader) refuseUnservedSearch(ctx context.Context, req issueops.ListRequest) error { - if req.Query == "" { - return nil +// this is the one handshake a tokened request costs, and a request without +// any of these members never asks for it. +func (r httpReader) refuseUnservedListFilters(ctx context.Context, req issueops.ListRequest) error { + var snap *apigen.ContextResponse + for _, f := range tokenedListFilters { + if !f.set(req) { + continue + } + if snap == nil { + var err error + if snap, err = r.store.snapshot(ctx); err != nil { + return err + } + } + if snap == nil || !slices.Contains(snap.Capabilities, f.token) { + return r.store.unsupportedCapability("Reader.List", f.token) + } } - snap, err := r.store.snapshot(ctx) - if err != nil { - return err - } - if snap != nil && slices.Contains(snap.Capabilities, wire.CapListSearch) { - return nil - } - return r.store.unsupportedCapability("Reader.List", wire.CapListSearch) + return nil } // Get serves getIssue, including the two landed include parameters that retired diff --git a/internal/httpclient/roles_test.go b/internal/httpclient/roles_test.go index 4aa353544..1f1e8cdf8 100644 --- a/internal/httpclient/roles_test.go +++ b/internal/httpclient/roles_test.go @@ -239,13 +239,13 @@ func TestReadRolesRefuseWhatTheWireCannotCarry(t *testing.T) { t.Fatalf("IssueReader(): %v", err) } - _, err = reader.List(ctx, issueops.ListRequest{IDFilter: "bd-1"}) + _, err = reader.List(ctx, issueops.ListRequest{SpecPrefix: "docs/"}) var inexpressible *InexpressibleError if !errors.As(err, &inexpressible) { - t.Fatalf("List with an IDFilter = %v, want *InexpressibleError", err) + t.Fatalf("List with a SpecPrefix = %v, want *InexpressibleError", err) } - if inexpressible.Refused.Row.Field != "IDFilter" { - t.Errorf("the refusal names %q, want IDFilter", inexpressible.Refused.Row.Field) + if inexpressible.Refused.Row.Field != "SpecPrefix" { + t.Errorf("the refusal names %q, want SpecPrefix", inexpressible.Refused.Row.Field) } if inexpressible.Refused.Row.SpecRow == "" || inexpressible.Refused.Row.Why == "" { t.Error("the refusal carries a ledger row with no reason or no citation") diff --git a/internal/httpclient/served_counter_test.go b/internal/httpclient/served_counter_test.go index 01fa3f7d4..7f11a4296 100644 --- a/internal/httpclient/served_counter_test.go +++ b/internal/httpclient/served_counter_test.go @@ -15,9 +15,9 @@ import ( // The Counter contract, run through the http client against a real bd serve. // // NOTHING IS PARKED HERE, and on this role that is a statement rather than a -// happy accident. Every case scopes itself with IDFilter — the count publishes -// `id` where the LISTING does not, which is why the Reader family parks nine -// cases on E-ListRequest.IDFilter and this one parks none — and every filter the +// happy accident. Every case scopes itself with IDFilter — the count has +// always published `id` (the listing gained it, behind issues.list.ids, with +// S15b) — and every filter the // contract reaches for is a parameter this operation carries. The plane cases in // particular cross whole: `include_infra` is one flag on the wire and one flag // on the request, and what it MEANS is four decisions the server's own role diff --git a/internal/httpclient/served_list_test.go b/internal/httpclient/served_list_test.go index 8cab1f837..751bee628 100644 --- a/internal/httpclient/served_list_test.go +++ b/internal/httpclient/served_list_test.go @@ -27,11 +27,11 @@ import ( // asserted through the same composition. // // Almost every List case in reader_contract.go scopes itself with IDFilter, and -// listIssues publishes no id parameter, so those cases are parked next door and -// their behaviors are pinned HERE instead — same assertions, scoped by a label, -// which the wire does carry. What is not restated here is anything the contract -// case actually runs: this file exists to close the gap the parking opened, not -// to grow a second contract. +// until S15b listIssues published no id parameter, so those cases were parked +// next door and their behaviors were pinned HERE instead — same assertions, +// scoped by a label. The id scope now rides `id` (issues.list.ids) and the +// contract cases run as written beside these; the label-scoped twins stay as +// the form of the same assertions a server without that token can still run. // listScope mints a label nothing else in the package uses. func listScope(name string) string { return servedIssuePrefix + "-ls-" + name } @@ -1472,7 +1472,7 @@ func TestServedReaderRefusesTheFiltersTheWireCannotCarry(t *testing.T) { field string req issueops.ListRequest }{ - {"IDFilter", issueops.ListRequest{IDFilter: listID("refuse", "a")}}, + {"SpecPrefix", issueops.ListRequest{SpecPrefix: "docs/"}}, {"ReadyFlag", issueops.ListRequest{ReadyFlag: true}}, {"Offset", issueops.ListRequest{Offset: 1}}, {"PinnedFlag", issueops.ListRequest{PinnedFlag: true}}, @@ -1519,9 +1519,8 @@ func seedListWisp(t *testing.T, ctx context.Context, c *servedComposition, id, s // TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope is // RunReaderListIncludeEphemeralMergesThePlanesIntoOneOrder with the id scope // swapped for a label one, which is the swap this whole file exists to make: -// the contract case scopes itself with IDFilter and listIssues publishes no id -// parameter (E-ListRequest.IDFilter), so the case is parked next door and its -// behavior is pinned here instead. +// the contract case scopes itself with IDFilter, which was parked next door +// until S15b put `id` on listIssues; both run now. // // WHAT IT IS ABOUT is not admission but ORDER. That the flag admits wisps at // all is already covered by RunReaderListDefaultExclusionsAndTheirOverrides, diff --git a/internal/httpclient/served_reader_test.go b/internal/httpclient/served_reader_test.go index be604e36d..46dd2a396 100644 --- a/internal/httpclient/served_reader_test.go +++ b/internal/httpclient/served_reader_test.go @@ -22,24 +22,23 @@ import ( // survives the JSON round trip. The seed handles are the reference store's, so // a filter this client dropped answers with rows no case seeded. // -// WHAT IS PARKED, and it is one cause with many names: `ListRequest.IDFilter` -// is the scoping tool almost every List case reaches for, and listIssues -// publishes no id parameter (E-ListRequest.IDFilter). Refusing it is correct — -// dropping it would answer about the whole workspace — but a case that scopes -// itself with it cannot run over this wire at all. Each parked case names the -// field that refused, and the refusal ITSELF is asserted below in -// TestServedReaderRefusesTheFiltersTheWireCannotCarry, so the parking never -// stands in for an unpinned behaviour. +// WHAT IS PARKED: `ListRequest.IDFilter` is the scoping tool almost every List +// case reaches for, and until S15b listIssues published no id parameter, so +// those cases parked on E-ListRequest.IDFilter. It now rides `id` behind +// issues.list.ids, which this in-process server serves, and they run as +// written. What still parks is a case whose SUBJECT (or one arm) is a member +// the wire refuses — Offset, ReadyFlag, PinnedFlag, ExcludeTypes, WispType, +// SkipCounts — and each names that member's row. The refusals THEMSELVES are +// asserted in TestServedReaderRefusesTheFiltersTheWireCannotCarry, so the +// parking never stands in for an unpinned behaviour. // readParkBead is the bead every READ-side park cites, and it is a different // constant from the write side's parkBead because it is a different decision // with a different retirement. // // The write parks retire when the wire grows the members it deliberately -// withholds, or never. These retire when the wire grows a way to SCOPE a -// listing by id — the upstream ask D8 row 1 records alongside the `sort` -// parameter — at which point almost all of them go at once. Folding the two -// into one constant would have said they move together, which they do not. +// withholds, or never. These retire one member at a time as the wire grows a +// parameter for it; the id scope that held most of them retired with S15b. const readParkBead = "ga-pcwtq" func servedReaderFixture(t *testing.T, name string) conformance.ReaderFixture { @@ -127,37 +126,29 @@ func TestServedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions(t *testing. } func TestServedReaderOffsetSkipsTheRowsBeforeThePage(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the List arm scopes itself with IDFilter, which listIssues publishes no parameter for; the Offset "+ - "refusal this wire answers with instead is pinned by TestServedReaderRefusesTheFiltersTheWireCannotCarry") + skipKnownDivergence(t, "E-ListRequest.Offset", readParkBead, + "the case's subject is Offset, which neither listIssues nor listReadyWork publishes (its id scope now "+ + "rides `id`); the Offset refusal this wire answers with instead is pinned by "+ + "TestServedReaderRefusesTheFiltersTheWireCannotCarry") conformance.RunReaderOffsetSkipsTheRowsBeforeThePage(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListDefaultExclusionsAndTheirOverrides(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "every arm scopes itself with IDFilter, and two of them additionally set PinnedFlag/NoPinnedFlag, "+ - "which listIssues publishes no parameters for") + skipKnownDivergence(t, "E-ListRequest.PinnedFlag", readParkBead, + "two arms set PinnedFlag/NoPinnedFlag, which listIssues publishes no parameters for (the id scope "+ + "every arm uses now rides `id`)") conformance.RunReaderListDefaultExclusionsAndTheirOverrides(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListRejectsATypeOutsideTheWorkspaceVocabulary(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the built-in-type arm scopes itself with IDFilter; the vocabulary refusal itself is served (the "+ - "server validates against its own authoritative vocabulary, which is what heals L7 for this query)") conformance.RunReaderListRejectsATypeOutsideTheWorkspaceVocabulary(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListNaturalNumericIDSortTrimsAfterTheFetch(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the natural-numeric order it pins is exercised over this wire "+ - "by TestServedReaderListAppliesTheClientSideDisplayOrder") conformance.RunReaderListNaturalNumericIDSortTrimsAfterTheFetch(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the skip-forward walk it pins is exercised over this wire by "+ - "TestServedReaderListKeysetPositionSkipsForward, which is the same assertion under a label scope") conformance.RunReaderListKeysetPositionResumesTheCreatedDescIDAscOrder(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -176,9 +167,6 @@ func TestServedReaderListReadyFlagRefusesAFilterItCannotCarry(t *testing.T) { } func TestServedReaderListEmptyPageIsWellFormed(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the List arm scopes itself with IDFilter; the empty-page shape is pinned over this wire by "+ - "TestServedReaderListEmptyPageIsWellFormedUnderALabelScope") conformance.RunReaderListEmptyPageIsWellFormed(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -190,8 +178,9 @@ func TestServedReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { } func TestServedReaderListMaxRowsIsHonored(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the cap it pins is honored over this wire and exercised by "+ + skipKnownDivergence(t, "E-ListRequest.Offset", readParkBead, + "one arm drives the cap behind an Offset, which listIssues publishes no parameter for (its id scope "+ + "now rides `id`); the cap is honored over this wire and exercised by "+ "TestServedReaderListMaxRowsBoundsWireRowsFetched") conformance.RunReaderListMaxRowsIsHonored(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -200,14 +189,11 @@ func TestServedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse(t *testin skipKnownDivergence(t, "E-ListRequest.SkipCounts", readParkBead, "the knob is DROPPED rather than refused (the wire hydrates the three cardinalities either way, and "+ "every text rendering of `bd list` sets it, so refusing would kill the default listing over http); "+ - "the case additionally scopes itself with IDFilter") + "its id scope rides `id` (issues.list.ids)") conformance.RunReaderListSkipCountsDropsTheCardinalitiesAndNothingElse(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the limit boundary under created order is pinned over this "+ - "wire by TestServedReaderListLimitBoundaryUnderALabelScope") conformance.RunReaderListLimitBoundaryUnderASortTheDatabaseCanExpress(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -218,83 +204,49 @@ func TestServedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters(t *testin } // TestServedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder is the plane -// contract client wave ga-mijra brought into range, and it parks for the same -// reason every List case here parks: it scopes itself with IDFilter, which -// listIssues publishes no parameter for. -// -// The behavior it asserts is pinned by -// TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope, which is -// the same fixture — alternating planes, every Limit cut, the keyset walk — -// with the id scope swapped for a label one, plus the plane-identity assertion +// contract client wave ga-mijra brought into range. It scopes itself with +// IDFilter, which rides listIssues' `id` behind issues.list.ids (S15b), so it +// runs as written; TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope +// is the same fixture under a label scope, plus the plane-identity assertion // the contract does not make. func TestServedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with ListRequest.IDFilter and listIssues publishes no id parameter; "+ - "the merged-plane order, the Limit cuts and the keyset walk are asserted under a label scope by "+ - "TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope") conformance.RunReaderListIncludeEphemeralMergesThePlanesIntoOneOrder(t, t.Context(), servedReaderFixture(t, "rdr")) } -// The FIVE remaining List parks, each named so the wiring lock counts the -// contract as accounted for rather than merely absent, and each deleted in one -// line the day its row retires. -// -// Three of them park on the id scope alone, which is this family's one cause. -// The other two would park even under a label scope, because their SUBJECT is a -// member the wire refuses — so each names that member instead. +// The id-scoped List cases that parked on E-ListRequest.IDFilter until S15b +// put `id` on listIssues (issues.list.ids). They run as written now; the two +// below them still park, because their SUBJECT is a member the wire refuses. func TestServedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the blocks-only rule it pins is exercised over this wire by "+ - "the cardinality assertions in TestServedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded, "+ - "whose `family` row carries a parent-child and a relates-to edge and still reports zero dependencies") conformance.RunReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter — its shadow rows share every timestamp with the scoped ones "+ - "and are excluded by the id set ALONE, which is the whole mechanism by which a widened position "+ - "is caught, so a label scope would not stand in for it") conformance.RunReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates(t, t.Context(), servedReaderFixture(t, "rdr")) } func TestServedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the keyset walk it pins is exercised over this wire by "+ - "TestServedReaderListKeysetPositionSkipsForward and by TestServedReaderListWalksMultiplePages, "+ - "both under a label scope") conformance.RunReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing(t, t.Context(), servedReaderFixture(t, "rdr")) } -// The priority-order walk parks on the same row as the created-order one -// above, with a narrower stand-in beside it: the sort=priority pushdown is -// pinned under a label scope by TestServedReaderListFallsBackToTheWalkWithoutTheCapability, -// and resuming a priority keyset position — the pager's keysetFilter -// (list_walk.go) deciding priority first and the (created_at, id) pair only at -// the position's own priority — by TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder. -// What neither covers is the WALK this case drives page after page over an -// oversized equal-key run, which stays unexercised over this wire until -// IDFilter (or a label-scoped rewrite) lets it run. +// The priority-order walk over an oversized equal-key run, page after page: +// the sort=priority pushdown (TestServedReaderListFallsBackToTheWalkWithoutTheCapability) +// and one resumed priority keyset position +// (TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder) do not +// cover the WALK, which this id-scoped case drives now that `id` is served. func TestServedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter, which listIssues publishes no parameter for, so its one-shot "+ - "read is refused before the walk begins; the sort=priority pushdown is pinned under a label scope by "+ - "TestServedReaderListFallsBackToTheWalkWithoutTheCapability, and one resumed priority keyset position by "+ - "TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder") conformance.RunReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing(t, t.Context(), servedReaderFixture(t, "rdr")) } -// TestServedReaderListMaxRowsBoundaryIsLimitPlusOffset names OFFSET rather than -// the id scope it also carries, because Offset is the one this case could not -// be rewritten around: the whole point is to drive the cap along the OFFSET +// TestServedReaderListMaxRowsBoundaryIsLimitPlusOffset parks on OFFSET: the +// whole point is to drive the cap along the OFFSET // axis with the limit held still, and listIssues publishes no offset parameter. // The LIMIT axis of the same boundary runs over this wire in // TestServedReaderListMaxRowsBoundsWireRowsFetched. func TestServedReaderListMaxRowsBoundaryIsLimitPlusOffset(t *testing.T) { skipKnownDivergence(t, "E-ListRequest.Offset", readParkBead, - "the case moves the cap along the Offset axis, and listIssues publishes no offset parameter (it "+ - "scopes itself with IDFilter besides); the cap's Limit axis is driven over this wire by "+ + "the case moves the cap along the Offset axis, and listIssues publishes no offset parameter; "+ + "the cap's Limit axis is driven over this wire by "+ "TestServedReaderListMaxRowsBoundsWireRowsFetched") conformance.RunReaderListMaxRowsBoundaryIsLimitPlusOffset(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -314,9 +266,6 @@ func TestServedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t } func TestServedReaderListStatusAcceptsACommaSeparatedORSet(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the OR set it pins is exercised over this wire by "+ - "TestServedReaderListStatusORSetUnderALabelScope, which drives the same three status shapes") conformance.RunReaderListStatusAcceptsACommaSeparatedORSet(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -337,18 +286,10 @@ func TestServedReaderListStatusAcceptsACommaSeparatedORSet(t *testing.T) { // their rows through one function. TestBriefListingsStampTheProjectionMarker // asserts that door-by-door; these assert what a projected row must contain. -// TestServedReaderListBriefDropsTheFreeFormTextAndNothingElse parks for the -// reason every List case here parks — it scopes itself with IDFilter, and -// listIssues publishes no id parameter — NOT for anything about the projection. -// -// The projection half is asserted over this wire by the ready sibling below, -// which drives the identical assertReaderBriefRow through the same encoder and -// the same row lift, under a label scope instead of an id one. +// TestServedReaderListBriefDropsTheFreeFormTextAndNothingElse scopes itself +// with IDFilter, which rides `id` (issues.list.ids), and the ready sibling below +// drives the identical assertReaderBriefRow under a label scope. func TestServedReaderListBriefDropsTheFreeFormTextAndNothingElse(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the case scopes itself with IDFilter; the projection it pins — the six heavy columns empty, the "+ - "identity fields and the three cardinalities intact, and IsLitePartial set — is asserted over this "+ - "wire by TestServedReaderReadyBriefDropsTheFreeFormTextAndNothingElse under a label scope") conformance.RunReaderListBriefDropsTheFreeFormTextAndNothingElse(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -442,8 +383,9 @@ func TestServedReaderGetPopulatesRowVersionForAGuardedWrite(t *testing.T) { } func TestServedReaderDoesNotMutateTheCallerRequest(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, - "the List leg of the tripwire scopes itself with IDFilter; the promise is exercised over this wire "+ - "by TestServedReaderDoesNotMutateTheCallerRequestOverTheWire") + skipKnownDivergence(t, "E-ListRequest.ExcludeTypes", readParkBead, + "the List leg of the tripwire sets ExcludeTypes, which listIssues publishes no parameter for (its id "+ + "scope now rides `id`); the promise is exercised over this wire by "+ + "TestServedReaderDoesNotMutateTheCallerRequestOverTheWire") conformance.RunReaderDoesNotMutateTheCallerRequest(t, t.Context(), servedReaderFixture(t, "rdr")) } diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 2b8fc11bd..5f7ee0131 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -163,6 +163,22 @@ const CapCountScope = "issues.count.scope" // `unknown_parameter`, and is never widened into the unsearched listing. const CapListSearch = "issues.list.search" +// CapListNoAssignee, CapListTitle and CapListIDs are the behavior +// capabilities announcing that listIssues honors `no_assignee`, `title` and the +// repeatable `id` (`bd list --no-assignee`, `--title`, `--id`; +// issueops.ListRequest's NoAssignee, TitleSearch and IDFilter), spelled exactly +// as httpapi's CapIssuesListNoAssignee, CapIssuesListTitle and CapListIDs. The +// Reader's List reads them off the handshake snapshot and refuses a request +// carrying a member LOCALLY, with a typed capability error naming that +// member's token, when the server does not advertise it — never a round trip +// an older server answers `unknown_parameter`, never a wider listing than +// asked for. +const ( + CapListNoAssignee = "issues.list.noAssignee" + CapListTitle = "issues.list.title" + CapListIDs = "issues.list.ids" +) + // CapBatchApplyLarge is the behavior capability announcing that issues.batchApply // accepts a batch larger than the compiled-in floor (100 items), up to the // raised ceiling (1000), spelled exactly as httpapi's constant of the same name @@ -392,6 +408,7 @@ var behaviorCapabilities = []string{ CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapCloseAutoCloseMolecule, CapListSearch, + CapListNoAssignee, CapListTitle, CapListIDs, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 966354325..aec04ef48 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -151,6 +151,15 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapListSearch != httpapi.CapIssuesListSearch { t.Errorf("CapListSearch = %q, server says %q", CapListSearch, httpapi.CapIssuesListSearch) } + if CapListNoAssignee != httpapi.CapIssuesListNoAssignee { + t.Errorf("CapListNoAssignee = %q, server says %q", CapListNoAssignee, httpapi.CapIssuesListNoAssignee) + } + if CapListTitle != httpapi.CapIssuesListTitle { + t.Errorf("CapListTitle = %q, server says %q", CapListTitle, httpapi.CapIssuesListTitle) + } + if CapListIDs != httpapi.CapListIDs { + t.Errorf("CapListIDs = %q, server says %q", CapListIDs, httpapi.CapListIDs) + } if CapCountScope != httpapi.CapIssuesCountScope { t.Errorf("CapCountScope = %q, server says %q", CapCountScope, httpapi.CapIssuesCountScope) } @@ -205,10 +214,12 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, httpapi.CapIssuesCloseAutoCloseMolecule, httpapi.CapIssuesListSearch, + httpapi.CapIssuesListNoAssignee, httpapi.CapIssuesListTitle, httpapi.CapListIDs, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit, "+ - "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule and issues.list.search tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule, issues.list.search, "+ + "issues.list.noAssignee, issues.list.title and issues.list.ids tokens", behaviorCapabilities) } } From e7ad85bcff871e9245ded8f8028b89d0ae669729 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 14 Oct 2026 03:59:35 +0000 Subject: [PATCH 04/76] fix(http): bd export over http through issueops.ReadExport (S21) (#7551) * fix(http): bd export over http through issueops.ReadExport (S21) bd export read through raw SearchIssues and the bulk label, dependency and comment reads, which the HTTP backend refuses (B3 #16), so gc's JSONL export order (gc bd export --all) failed on every remote scope. issueops.ReadExport composes role reads every deployed server publishes: an unlimited Reader.List for rows, labels and counts, EdgeReader.ReadEdges for dependency records, and Reader.Get with comments for rows that have any. The remote route feeds it into the same runExportFromSource the direct and proxied routes use. A failed read fails the export. gc-export-all passes against the reference profile; mc-promote stays pending (S21b) with its exact reason: it needs a new server capability. Co-Authored-By: Claude Opus 5.5 * fix(export): read remote memories from the memory plane, pin unmarked no-history rows (bd-o4qlui) Review fixes for #7551, attempts 1 to 3. Attempt 3. No test reached roleExportSource.ExportMemories's error path. A unit test now drives it with a fake store: a memory role that cannot be reached, and a listMemories that fails, each end the export with that error and no map. Rewriting either branch to degrade to an empty map would bring back a remote `bd export --all` that exits 0 with no memory in it. Declined in attempt 3, for finalize to file: an unlimited HTTP listing that stops on a has_more page with no next_cursor returns the rows it has with HasMore false, so ReadExport's refusal cannot fire over HTTP. Only a server that breaks the spec's has_more/next_cursor pairing gets there; this repo's server always sends a cursor with a non-empty has_more page. The walker (fetchIssuePages) is unchanged since #7247 and shared with bd list, so the fix needs its own PR that checks bd list's callers first. The remote route also has no single snapshot: a comment changed between List and Get can make the written count disagree with the bodies kept. That belongs in one follow-up with the bounded retry declined in attempt 1, as a count/body consistency check. The --include-infra e2e leg stays declined. The wire does carry the knob: the client sends include_infra and include_templates on listIssues, and the server decodes both. The GetInfraTypes nil-store guard stays declined, and auto-export's own GetAllConfig memory read predates this PR. Attempt 2. A remote export dropped every memory. Over HTTP, `bd export --all` and `--include-memories` exited 0 without a memory record, where the base refused with exit 1. The memory lines came from GetAllConfig, which over HTTP is listSettings, and the server's settings enumeration withholds every kv.* row, memories included. gc's JSONL export runs `gc bd export --all`, so its archive lost them without a word. The exportSource seam now asks each source for its memories (ExportMemories), not for the whole config table. The local and proxied sources still take them from their config table. The remote source reads the served memory plane, store.Memories().List (listMemories), which the deployed bd 1.3.0 server advertises as memories.list. httpMemories.List refuses a page that reports has_more: ListResult has no cursor, so a truncated plane would pass for the whole one. The memory error now says "failed to read memories"; nothing in beads or gascity matches the old text. TestE2E_ExportOverHTTP seeds two memories with `bd remember` over HTTP. It requires exactly those two in --all, in --include-memories and in the local re-export of the imported file, and none in the default export. A unit case pins the has_more refusal. The CHANGELOG entry says where remote memories come from, so its "Nothing is dropped" holds. The WispPlaneIDs doc no longer says the listing refuses an id set, which stops being true once a server advertises issues.list.ids (#7550). Attempt 1. The remote route's WispPlaneIDs doc and the CHANGELOG had the effect of the missing wisp_plane marker backwards. They said import then routes a no-history row by its flags. A current import does the opposite: it pins an unmarked no-history row to the durable plane (applyImportWispPlane). So a genuine no-history wisp in a remote export, which gc's session, wait and order beads are by default, re-imports onto the Dolt-versioned issues plane, and gc's JSONL archive keeps it. The route still stamps nothing. No served read publishes plane membership: listIssues rows carry no plane field, however the listing is scoped. Inferring the marker from the flags would re-plane a durable row that still carries the flag and drop its relations (bd-r9uce), and refusing the export would re-break gc's remote JSONL export. The doc and the CHANGELOG now state the real consequence, and serving the membership is bd-1lbvas. TestE2E_ExportOverHTTP pins it. A no-history row is exported without the marker by --all and by the default export. A fresh local workspace imports it onto the durable plane, against a locally created control row that the local route does mark. Attempt 1 minors and nits: ReadExport refuses a listing page marked HasMore, as the uow source does, and unit cases pin that and a failed comment read. The exportSource interface doc names ExportIssues and its contract. The export error now says "failed to read the export set" rather than "failed to search issues"; nothing in beads or gascity matches the old text. roleExportSource.GetInfraTypes says it is unused on this route. Declined, for finalize to file: a bounded retry of ReadExport when a row vanishes between the reads (a design change; today's fatal behavior is pinned), and e2e rows for pinned, gate, template and infra-typed issues, including an --include-infra leg over HTTP. Also declined: a nil-store guard on roleExportSource.GetInfraTypes. The method is dead on this route, and the source's other store readers (ExportIssues, GetConfig, ExportMemories) dereference the store unguarded too. Validation, attempt 3: git diff --check, the go.mod-pinned gofmt, go vet, gazelle (no change) and the go_srcs check. The new test passed with no skips. Three mutants were killed. An empty map with no error on a failed listing, and the same on an unreachable role, each fail their own subtest. An empty map returned beside the listing error fails the no-map assertion; export.go discards the map on error today, so that case pins the interface's "fail rather than return a partial set" contract, not visible output. Validation, attempt 2: git diff --check, gofmt and go vet. The httpclient memory tests passed, 23 of them: 4 units and 19 served. So did TestE2E_ExportOverHTTP on embedded Dolt with BEADS_HTTP_TEST_REQUIRED=1 and no skips. Two mutants were killed: a remote source that reads the settings listing fails 3 e2e assertions, and dropping the has_more refusal fails the unit case. Export parity passed with no skips on the classic (memory determinism, byte stability), embedded and proxied tiers, including the proxied-versus- classic byte identity with memories. The PR is now behind main, which added #7549 and #7550, and conflicts with it textually in CHANGELOG.md, backend/http/BUILD.bazel and issueops/BUILD.bazel. With those resolved by union, the merge with main e6a891b900 builds and vets. On that merge, in CI's pinned-shard layout, the served http tier passed 49 of 49, including #7550's list-scope e2e. The command corpus ran 128 shapes: 113 pass, 3 allowed, 12 pending and 0 fail. gc-export-all passes without its pending mark, the one shape that moved. The issueops ReadExport and ListGates tests passed, as did the embedded and proxied export tiers. Validation, attempt 1: the issueops ReadExport tests, with two mutants killed; TestE2E_ExportOverHTTP with a flag-stamping WispPlaneIDs mutant killed on three assertions; the cmd/bd export tests in a scrubbed env. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 17 ++ backend/http/BUILD.bazel | 1 + backend/http/corpus/testdata/corpus.yaml | 3 +- backend/http/e2e_export_test.go | 203 +++++++++++++++++++++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/export.go | 117 +++++++------ cmd/bd/export_source.go | 158 ++++++++++++++++-- cmd/bd/export_source_role_test.go | 64 +++++++ internal/httpclient/memories.go | 10 ++ internal/httpclient/write_roles_test.go | 18 ++ issueops/BUILD.bazel | 2 + issueops/export.go | 130 +++++++++++++++ issueops/export_test.go | 160 ++++++++++++++++++ 13 files changed, 813 insertions(+), 71 deletions(-) create mode 100644 backend/http/e2e_export_test.go create mode 100644 cmd/bd/export_source_role_test.go create mode 100644 issueops/export.go create mode 100644 issueops/export_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b0209e93..580099cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -459,6 +459,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--all`: the same rows the raw filter selected, on both planes. `--limit 0` is unlimited, as before. The bead-scoped form (`bd gate list `) is unchanged. +- **`bd export` works over a connected HTTP workspace** (S21). It read + through the raw `SearchIssues` method and the bulk label, dependency and + comment reads, which the HTTP backend refuses, so gc's 15-minute JSONL + export order (`gc bd export --all`) failed on every remote scope. On a remote + backend the read set now comes from one library entry, `issueops.ReadExport`, + composed over role reads every deployed server publishes: an unlimited + `Reader.List` for the rows, their labels and counts, `EdgeReader.ReadEdges` + for the dependency records, and `Reader.Get` with comments for each row + that has any. The JSONL is written by the same code as on every other + route. Memories (`--all`, `--include-memories`) come from the served memory + plane (`listMemories`), not the settings listing, which withholds every + `kv.*` row and so exported none. A failed read fails the export, never + shortens it. A remote export does not yet stamp the `wisp_plane` marker, + because the server publishes no plane membership. So a current `bd import` + of a remote export puts genuine no-history wisps on the durable issues + plane instead of the wisps plane, and gc's JSONL archive keeps them. + Nothing is dropped. - **An update of a template is now refused for every caller, not only `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index dc812c067..045b68860 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -32,6 +32,7 @@ go_test( srcs = [ "attach_test.go", "e2e_cli_test.go", + "e2e_export_test.go", "e2e_list_filters_test.go", "e2e_mol_test.go", "e2e_recompute_blocked_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 7eb95caeb..9c3d2e485 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -434,7 +434,6 @@ shapes: source: gc core pack reaper (gc bd export --all -o F) args: [export, --all, -o, "{file:export.jsonl}"] compare: exit - pending: {slice: "S21", reason: "export --all is not wired over http (B3 #16)"} # ================================================================ writes # ---------------------------------------------------------------- create @@ -686,7 +685,7 @@ shapes: source: mc mol-normalized-issue-pr-v1 (promote --reason) args: [promote, e2e-wisp-p, --reason, "keep it"] compare: exit - pending: {slice: "S21", reason: "promote is not wired over http (B3 #16)"} + pending: {slice: "S21b", reason: "promote has no served operation (B3 #16): the http Store stubs PromoteFromEphemeral, updateIssue's patch does not publish persistence (W-IssuePatch.Persistence), and no server has a promote route; it needs a new additive capability, then SRV-1"} - id: mc-mol-burn source: mc review fixtures (mol burn --force) args: [mol, burn, e2e-wisp-m, --force] diff --git a/backend/http/e2e_export_test.go b/backend/http/e2e_export_test.go new file mode 100644 index 000000000..6b5a7e804 --- /dev/null +++ b/backend/http/e2e_export_test.go @@ -0,0 +1,203 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "maps" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +type exportLine struct { + Type string `json:"_type"` + ID string `json:"id"` + Ephemeral bool `json:"ephemeral"` + NoHistory bool `json:"no_history"` + WispPlane *bool `json:"wisp_plane"` + Labels []string `json:"labels"` + CommentCount int `json:"comment_count"` + DepCount int `json:"dependency_count"` + Dependencies []struct { + IssueID string `json:"issue_id"` + DependsOnID string `json:"depends_on_id"` + Type string `json:"type"` + } `json:"dependencies"` + Comments []struct { + Text string `json:"text"` + } `json:"comments"` + // Key and Value are a memory record's fields. + Key string `json:"key"` + Value string `json:"value"` +} + +// plane renders the explicit wisps-plane marker: "absent", "true" or "false". +func (l exportLine) plane() string { + if l.WispPlane == nil { + return "absent" + } + return strconv.FormatBool(*l.WispPlane) +} + +func readExportRecords(t *testing.T, path string) []exportLine { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read export: %v", err) + } + var out []exportLine + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + var rec exportLine + if err := json.Unmarshal([]byte(line), &rec); err != nil { + t.Fatalf("parse export line %q: %v", line, err) + } + out = append(out, rec) + } + return out +} + +func readExportLines(t *testing.T, path string) map[string]exportLine { + t.Helper() + out := map[string]exportLine{} + for _, rec := range readExportRecords(t, path) { + if rec.Type == "issue" { + out[rec.ID] = rec + } + } + return out +} + +// readExportMemories returns an export's memory records, value by key. +func readExportMemories(t *testing.T, path string) map[string]string { + t.Helper() + out := map[string]string{} + for _, rec := range readExportRecords(t, path) { + if rec.Type == "memory" { + out[rec.Key] = rec.Value + } + } + return out +} + +func wantExportMemories(t *testing.T, export string, got, want map[string]string) { + t.Helper() + if !maps.Equal(got, want) { + t.Errorf("%s memories = %v, want %v", export, got, want) + } +} + +// TestE2E_ExportOverHTTP is S21. `bd export` read through the raw SearchIssues +// method and the bulk relation reads, which the http backend refuses, so gc's +// JSONL-export order failed on every remote scope. It now composes served role +// reads (issueops.ReadExport): the rows with their labels and counts, every +// row's dependency records and the comment bodies. Memories come from the +// served memory plane (listMemories): the settings enumeration withholds every +// kv.* row, so a remote export that read them there carried none. +// +// The no-history row pins the one place this route still differs from the +// local ones. No served read publishes plane membership, so the row is +// exported without the wisp_plane marker, and a current import pins it to the +// durable issues plane: the data-safe side, since a wrong marker would drop a +// durable row's relations (bd-r9uce). Serving the membership (bd-1lbvas) +// flips those assertions: the row is then marked and re-imports as a wisp. +func TestE2E_ExportOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + blocked := createE2EIssue(t, bd, "blocked work") + blocker := createE2EIssue(t, bd, "blocker") + mustRun(t, bd, "dep", "add", blocked, blocker, "--json") + mustRun(t, bd, "label", "add", blocked, "lane:x", "--json") + mustRun(t, bd, "comments", "add", blocked, "an export comment") + closed := createE2EIssue(t, bd, "closed work") + mustRun(t, bd, "close", closed, "--json") + wisp := firstJSONField(t, mustRun(t, bd, "create", "a wisp", "--ephemeral", "--json").stdout, "id") + noHistory := firstJSONField(t, mustRun(t, bd, "create", "a no-history wisp", "--no-history", "--json").stdout, "id") + memories := map[string]string{ + "export-alpha": "remembered over http", + "export-beta": "a second memory", + } + for key, content := range memories { + mustRun(t, bd, "remember", content, "--key", key) + } + + dir := t.TempDir() + all := filepath.Join(dir, "all.jsonl") + mustRun(t, bd, "export", "--all", "-o", all) + wantExportMemories(t, "bd export --all", readExportMemories(t, all), memories) + got := readExportLines(t, all) + for _, id := range []string{blocked, blocker, closed, wisp, noHistory} { + if _, ok := got[id]; !ok { + t.Errorf("bd export --all is missing %s; got %d issues", id, len(got)) + } + } + rec := got[blocked] + if len(rec.Dependencies) != 1 || rec.Dependencies[0].DependsOnID != blocker || rec.DepCount != 1 { + t.Errorf("%s dependencies = %+v (count %d), want one edge onto %s", blocked, rec.Dependencies, rec.DepCount, blocker) + } + if len(rec.Labels) != 1 || rec.Labels[0] != "lane:x" { + t.Errorf("%s labels = %v, want [lane:x]", blocked, rec.Labels) + } + if rec.CommentCount != 1 || len(rec.Comments) != 1 || rec.Comments[0].Text != "an export comment" { + t.Errorf("%s comments = %+v (count %d), want the one comment", blocked, rec.Comments, rec.CommentCount) + } + if !got[wisp].Ephemeral { + t.Errorf("%s exported without ephemeral: %+v", wisp, got[wisp]) + } + if rec := got[noHistory]; !rec.NoHistory || rec.plane() != "absent" { + t.Errorf("%s exported as no_history=%v wisp_plane=%s, want no_history with no marker", noHistory, rec.NoHistory, rec.plane()) + } + + // The default export keeps every status and the no-history row, and drops + // the true wisp. + def := filepath.Join(dir, "default.jsonl") + mustRun(t, bd, "export", "-o", def) + gotDef := readExportLines(t, def) + if _, ok := gotDef[wisp]; ok { + t.Errorf("bd export (default) exported the wisp %s", wisp) + } + for _, id := range []string{blocked, blocker, closed, noHistory} { + if _, ok := gotDef[id]; !ok { + t.Errorf("bd export (default) is missing %s", id) + } + } + if rec := gotDef[noHistory]; rec.plane() != "absent" { + t.Errorf("bd export (default) stamped %s wisp_plane=%s, want no marker", noHistory, rec.plane()) + } + // Memories are opt-in: absent by default, present with --include-memories. + wantExportMemories(t, "bd export (default)", readExportMemories(t, def), map[string]string{}) + withMemories := filepath.Join(dir, "memories.jsonl") + mustRun(t, bd, "export", "--include-memories", "-o", withMemories) + wantExportMemories(t, "bd export --include-memories", readExportMemories(t, withMemories), memories) + + // Import the remote export into a fresh local workspace and export it + // again there. The local route stamps wisp_plane from table membership, + // which the locally created control row proves, so the imported row's + // missing marker means it sits in the durable issues table. + bin := buildBD(t) + localDir := t.TempDir() + local := func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, localDir, []string{"BEADS_ACTOR=worker"}, args...) + } + mustRun(t, local, "init", "--prefix", "e2e", "--quiet") + mustRun(t, local, "import", "-i", all) + control := firstJSONField(t, mustRun(t, local, "create", "a local no-history wisp", "--no-history", "--json").stdout, "id") + back := filepath.Join(dir, "reexport.jsonl") + mustRun(t, local, "export", "--all", "-o", back) + gotBack := readExportLines(t, back) + if rec, ok := gotBack[control]; !ok || rec.plane() != "true" { + t.Fatalf("local control %s exported with wisp_plane=%s (present=%v), want true", control, rec.plane(), ok) + } + wantExportMemories(t, "the local re-export", readExportMemories(t, back), memories) + if rec, ok := gotBack[noHistory]; !ok { + t.Errorf("the local re-export is missing the imported %s", noHistory) + } else if !rec.NoHistory || rec.plane() != "absent" { + t.Errorf("imported %s re-exported as no_history=%v wisp_plane=%s, want no_history on the durable plane (no marker)", noHistory, rec.NoHistory, rec.plane()) + } +} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 4a5e6605a..043395ee2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -780,6 +780,7 @@ go_test( "export_exclude_owner_test.go", "export_max_rows_gate_test.go", "export_proxied_integration_test.go", + "export_source_role_test.go", "export_source_wisp_subset_test.go", "export_test.go", "external_policy_direct_integration_test.go", diff --git a/cmd/bd/export.go b/cmd/bd/export.go index 69bba6405..e616b45bc 100644 --- a/cmd/bd/export.go +++ b/cmd/bd/export.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) var exportCmd = &cobra.Command{ @@ -100,6 +101,11 @@ func runExport(cmd *cobra.Command, args []string) error { return err } + if storeIsRemoteBackend(store) { + // A remote backend serves no raw search or bulk relation read; its + // export is composed from served role reads (issueops.ReadExport). + return runExportFromSource(ctx, &roleExportSource{}) + } return runExportFromSource(ctx, storeExportSource{}) } @@ -129,49 +135,9 @@ func runExportFromSource(ctx context.Context, src exportSource) error { w = os.Stdout } - // Build filter for issues table. Export all statuses by default. - // Opt out of BEADS_MAX_ROWS (designer §4.1) — export is a data-integrity - // path and must never abort partway through an export run. - filter := types.IssueFilter{ - Limit: 0, - MaxRows: 0, - MaxRowsSource: "", - } - - // Exclude infra types by default (agents, roles, messages). - if !exportAll && !exportIncludeInfra { - var infraTypes []string - infraSet := src.GetInfraTypes(ctx) - if len(infraSet) > 0 { - for t := range infraSet { - infraTypes = append(infraTypes, t) - } - } - if len(infraTypes) == 0 { - infraTypes = domain.DefaultInfraTypes() - } - for _, t := range infraTypes { - filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t)) - } - } - - // Exclude templates by default - if !exportAll { - isTemplate := false - filter.IsTemplate = &isTemplate - } - - // Exclude ephemeral wisps by default — they are private/transient and - // must not reach git history or external integrations (GH#3649). - // --all overrides to include everything. - if !exportAll { - persistentOnly := false - filter.Ephemeral = &persistentOnly - } - - issues, err := src.SearchIssues(ctx, "", filter) + issues, err := src.ExportIssues(ctx, issueops.ExportRequest{All: exportAll, IncludeInfra: exportIncludeInfra}) if err != nil { - return HandleErrorRespectJSON("failed to search issues: %v", err) + return HandleErrorRespectJSON("failed to read the export set: %v", err) } // Scrub test/pollution records if requested @@ -271,26 +237,21 @@ func runExportFromSource(ctx context.Context, src exportSource) error { // Memories may contain sensitive agent context and are excluded by default. memoryCount := 0 if (exportIncludeMemories || exportAll) && !exportNoMemories { - allConfig, err := src.GetAllConfig(ctx) + memories, err := src.ExportMemories(ctx) if err != nil { - return HandleErrorRespectJSON("failed to read config for memories: %v", err) + return HandleErrorRespectJSON("failed to read memories: %v", err) } - fullPrefix := kvPrefix + memoryPrefix // Sort keys for deterministic output order (GH#3474). - var memKeys []string - for k := range allConfig { - if strings.HasPrefix(k, fullPrefix) { - memKeys = append(memKeys, k) - } + memKeys := make([]string, 0, len(memories)) + for userKey := range memories { + memKeys = append(memKeys, userKey) } sort.Strings(memKeys) - for _, k := range memKeys { - v := allConfig[k] - userKey := strings.TrimPrefix(k, fullPrefix) + for _, userKey := range memKeys { record := map[string]string{ "_type": "memory", "key": userKey, - "value": v, + "value": memories[userKey], } data, err := json.Marshal(record) if err != nil { @@ -328,6 +289,54 @@ func runExportFromSource(ctx context.Context, src exportSource) error { return nil } +// exportIssueFilter is the storage filter the store-shaped export sources +// search with: every status, infra types excluded unless --all or +// --include-infra, templates and true wisps excluded unless --all. The +// role-backed source asks the same question through issueops.ReadExport. +func exportIssueFilter(ctx context.Context, src exportSource, req issueops.ExportRequest) types.IssueFilter { + // Build filter for issues table. Export all statuses by default. + // Opt out of BEADS_MAX_ROWS (designer §4.1) — export is a data-integrity + // path and must never abort partway through an export run. + filter := types.IssueFilter{ + Limit: 0, + MaxRows: 0, + MaxRowsSource: "", + } + + // Exclude infra types by default (agents, roles, messages). + if !req.All && !req.IncludeInfra { + var infraTypes []string + infraSet := src.GetInfraTypes(ctx) + if len(infraSet) > 0 { + for t := range infraSet { + infraTypes = append(infraTypes, t) + } + } + if len(infraTypes) == 0 { + infraTypes = domain.DefaultInfraTypes() + } + for _, t := range infraTypes { + filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t)) + } + } + + // Exclude templates by default + if !req.All { + isTemplate := false + filter.IsTemplate = &isTemplate + } + + // Exclude ephemeral wisps by default — they are private/transient and + // must not reach git history or external integrations (GH#3649). + // --all overrides to include everything. + if !req.All { + persistentOnly := false + filter.Ephemeral = &persistentOnly + } + + return filter +} + // exportIssueRecord wraps IssueWithCounts with a _type discriminator so that // every line in the JSONL export is self-describing. Memory lines already // carry "_type":"memory"; this gives issue lines "_type":"issue". (GH#3271) diff --git a/cmd/bd/export_source.go b/cmd/bd/export_source.go index df4e4acfd..8b28b6eaa 100644 --- a/cmd/bd/export_source.go +++ b/cmd/bd/export_source.go @@ -4,11 +4,14 @@ import ( "context" "errors" "fmt" + "strings" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dberrors" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" ) // exportSource abstracts the storage reads `bd export` performs, so the one @@ -30,16 +33,17 @@ type exportSource interface { // hardcoded defaults). A nil/empty result makes the caller fall back to // domain.DefaultInfraTypes(), mirroring the pre-seam behavior. GetInfraTypes(ctx context.Context) map[string]bool - // SearchIssues returns the full, untruncated result set for the export - // filter (Limit=0, MaxRows=0). Implementations MUST fail rather than - // return a truncated page — export is a data-integrity path. - SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) + // ExportIssues returns the full, untruncated export set for req. + // Implementations MUST fail rather than return a truncated or shortened + // set — export is a data-integrity path. + ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) // GetConfig reads one config key from the database (used for the // export.exclude_owners owner-filter keys). GetConfig(ctx context.Context, key string) (string, error) - // GetAllConfig reads the whole config table (used to extract kv.memory.* - // rows when memories are exported). - GetAllConfig(ctx context.Context) (map[string]string, error) + // ExportMemories returns every persistent memory (`bd remember`), keyed + // by the user key without the kv.memory. prefix. Like ExportIssues, it + // MUST fail rather than return a partial set. + ExportMemories(ctx context.Context) (map[string]string, error) // LoadExportRelations bulk-loads labels, dependency records, comments, // comment counts, and dependency counts for the searched issues. LoadExportRelations(ctx context.Context, issues []*types.Issue) (exportRelations, error) @@ -89,8 +93,8 @@ func (storeExportSource) GetInfraTypes(ctx context.Context) map[string]bool { return store.GetInfraTypes(ctx) } -func (storeExportSource) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { - return store.SearchIssues(ctx, query, filter) +func (src storeExportSource) ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) { + return store.SearchIssues(ctx, "", exportIssueFilter(ctx, src, req)) } func (storeExportSource) GetConfig(ctx context.Context, key string) (string, error) { @@ -100,8 +104,24 @@ func (storeExportSource) GetConfig(ctx context.Context, key string) (string, err return store.GetConfig(ctx, key) } -func (storeExportSource) GetAllConfig(ctx context.Context) (map[string]string, error) { - return store.GetAllConfig(ctx) +func (storeExportSource) ExportMemories(ctx context.Context) (map[string]string, error) { + allConfig, err := store.GetAllConfig(ctx) + if err != nil { + return nil, err + } + return memoriesFromConfig(allConfig), nil +} + +// memoriesFromConfig extracts the persistent memories from a whole config +// table: its kv.memory.* rows, keyed by the user key. +func memoriesFromConfig(allConfig map[string]string) map[string]string { + memories := map[string]string{} + for k, v := range allConfig { + if userKey, ok := strings.CutPrefix(k, kvPrefix+memoryPrefix); ok { + memories[userKey] = v + } + } + return memories } func (storeExportSource) LoadExportRelations(ctx context.Context, issues []*types.Issue) (exportRelations, error) { @@ -177,8 +197,9 @@ func (s *uowExportSource) GetInfraTypes(ctx context.Context) map[string]bool { return infra } -func (s *uowExportSource) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { - page, err := s.uw.IssueUseCase().SearchIssues(ctx, query, filter) +func (s *uowExportSource) ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) { + filter := exportIssueFilter(ctx, s, req) + page, err := s.uw.IssueUseCase().SearchIssues(ctx, "", filter) if err != nil { return nil, err } @@ -196,8 +217,12 @@ func (s *uowExportSource) GetConfig(ctx context.Context, key string) (string, er return s.uw.ConfigUseCase().GetConfig(ctx, key) } -func (s *uowExportSource) GetAllConfig(ctx context.Context) (map[string]string, error) { - return s.uw.ConfigUseCase().GetAllConfig(ctx) +func (s *uowExportSource) ExportMemories(ctx context.Context) (map[string]string, error) { + allConfig, err := s.uw.ConfigUseCase().GetAllConfig(ctx) + if err != nil { + return nil, err + } + return memoriesFromConfig(allConfig), nil } // LoadExportRelations reproduces the classic bulk loaders over the @@ -405,3 +430,106 @@ func mergeExportMap[V any](dst, src map[string]V) { dst[k] = v } } + +// roleExportSource is the export read set of a remote backend, which serves +// no raw search and no bulk relation read. The issues and every relation come +// from ONE library entry, issueops.ReadExport, composed over role reads every +// deployed server publishes (listIssues, listDependencies, getIssue). The +// JSONL is then built by the same runExportFromSource as on every other route. +type roleExportSource struct { + rows map[string]issueops.ExportRow +} + +// GetInfraTypes is unused on this route: only exportIssueFilter, the filter of +// the store-shaped sources, reads it. ReadExport's listing excludes the infra +// set the server resolves for its own workspace. +func (s *roleExportSource) GetInfraTypes(ctx context.Context) map[string]bool { + return store.GetInfraTypes(ctx) +} + +func (s *roleExportSource) ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) { + rd, err := store.IssueReader() + if err != nil { + return nil, err + } + edges, err := store.EdgeReader() + if err != nil { + return nil, err + } + rows, err := issueops.ReadExport(ctx, rd, edges, req) + if err != nil { + return nil, err + } + s.rows = make(map[string]issueops.ExportRow, len(rows)) + issues := make([]*types.Issue, 0, len(rows)) + for _, row := range rows { + s.rows[row.Issue.ID] = row + issues = append(issues, row.Issue) + } + return issues, nil +} + +func (s *roleExportSource) GetConfig(ctx context.Context, key string) (string, error) { + return store.GetConfig(ctx, key) +} + +// ExportMemories reads the served memory plane (listMemories). The config +// table cannot stand in for it on this route: the server's settings +// enumeration withholds every kv.* row, memories included, so reading it +// exported no memory at all. +func (s *roleExportSource) ExportMemories(ctx context.Context) (map[string]string, error) { + memories, err := store.Memories() + if err != nil { + return nil, err + } + res, err := memories.List(ctx, memoryops.ListRequest{}) + if err != nil { + return nil, err + } + return res.Memories, nil +} + +// LoadExportRelations hands back what ReadExport already read; it issues no +// request of its own. +func (s *roleExportSource) LoadExportRelations(_ context.Context, issues []*types.Issue) (exportRelations, error) { + rel := exportRelations{ + labels: map[string][]string{}, + deps: map[string][]*types.Dependency{}, + comments: map[string][]*types.Comment{}, + commentCounts: map[string]int{}, + depCounts: map[string]*types.DependencyCounts{}, + } + for _, issue := range issues { + row, ok := s.rows[issue.ID] + if !ok { + continue + } + if len(row.Issue.Labels) > 0 { + rel.labels[issue.ID] = row.Issue.Labels + } + if len(row.Issue.Dependencies) > 0 { + rel.deps[issue.ID] = row.Issue.Dependencies + } + if len(row.Issue.Comments) > 0 { + rel.comments[issue.ID] = row.Issue.Comments + } + rel.commentCounts[issue.ID] = row.CommentCount + rel.depCounts[issue.ID] = &types.DependencyCounts{DependencyCount: row.DependencyCount, DependentCount: row.DependentCount} + } + return rel, nil +} + +// WispPlaneIDs answers nothing: no served read publishes plane membership +// (listIssues rows carry no plane field, however the listing is scoped), so a +// remote export stamps no row wisp_plane. A current import pins an +// unmarked no-history row to the durable plane (applyImportWispPlane). That +// is right for a durable row that still carries the flag (a legacy promotion, +// or an earlier unmarked import) and wrong for a genuine no-history wisp, +// which imports onto the Dolt-versioned issues plane. It is the same data-safe +// degradation as a store without storeWispPartitioner: a missing marker +// mis-planes a row, while a wrong one re-planes a durable row and drops its +// relations (bd-r9uce), so the marker is never inferred from flags. Serving +// the membership is bd-1lbvas. +func (s *roleExportSource) WispPlaneIDs(context.Context, []string) (map[string]bool, error) { + return nil, nil +} diff --git a/cmd/bd/export_source_role_test.go b/cmd/bd/export_source_role_test.go new file mode 100644 index 000000000..2f389d36c --- /dev/null +++ b/cmd/bd/export_source_role_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/memoryops" +) + +// memoriesOnlyStore is a remote backend reduced to the one read +// roleExportSource.ExportMemories makes: it embeds a nil DoltStorage and +// answers only the Memories role accessor. +type memoriesOnlyStore struct { + storage.DoltStorage + memories memoryops.Memories + err error +} + +func (s *memoriesOnlyStore) Memories() (memoryops.Memories, error) { return s.memories, s.err } + +// failingMemoryList is a memory plane whose listing (listMemories) fails. +type failingMemoryList struct { + memoryops.Memories + err error +} + +func (l failingMemoryList) List(context.Context, memoryops.ListRequest) (memoryops.ListResult, error) { + return memoryops.ListResult{}, l.err +} + +// TestRoleExportSourceExportMemoriesFailsTheExport pins exportSource's "fail +// rather than return a partial set" on the remote route: a memory plane that +// cannot be reached or listed ends the export with that error, never with an +// empty map. Degrading to empty would bring back what reading memories from +// the settings listing did, a remote `bd export --all` that exits 0 with no +// memory in it. +func TestRoleExportSourceExportMemoriesFailsTheExport(t *testing.T) { + oldStore := store + t.Cleanup(func() { store = oldStore }) + + unreachable := errors.New("memories: no transport") + listFailed := errors.New("listMemories: 503 Service Unavailable") + for _, tc := range []struct { + name string + store *memoriesOnlyStore + want error + }{ + {"the role is unavailable", &memoriesOnlyStore{err: unreachable}, unreachable}, + {"the listing fails", &memoriesOnlyStore{memories: failingMemoryList{err: listFailed}}, listFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + store = tc.store + got, err := (&roleExportSource{}).ExportMemories(context.Background()) + if !errors.Is(err, tc.want) { + t.Fatalf("ExportMemories error = %v, want %v", err, tc.want) + } + if got != nil { + t.Fatalf("ExportMemories returned %v with its error, want no map", got) + } + }) + } +} diff --git a/internal/httpclient/memories.go b/internal/httpclient/memories.go index 986f6a8f4..28778cde7 100644 --- a/internal/httpclient/memories.go +++ b/internal/httpclient/memories.go @@ -5,6 +5,7 @@ package httpclient import ( "context" "errors" + "fmt" "strings" "github.com/steveyegge/beads/internal/httpapi/apigen" @@ -95,11 +96,20 @@ func (m *httpMemories) Forget(ctx context.Context, req memoryops.ForgetRequest) // The folding is the ROLE's, so it happens server-side: the raw term goes on the // wire as `search` and the server's own role lowercases both sides. Folding it // here as well would be a second implementation of a rule that has one. +// +// A TRUNCATED PAGE IS REFUSED. The envelope is paginated but ListResult has no +// cursor, so a page reporting has_more would pass a partial plane off as the +// whole one, and `bd export` would drop the rest without a word. No v0 server +// truncates; this keeps the day one does from becoming a silent loss. func (m *httpMemories) List(ctx context.Context, req memoryops.ListRequest) (memoryops.ListResult, error) { page, err := m.wire.ListMemories(ctx, req.Search) if err != nil { return memoryops.ListResult{}, err } + if page.HasMore { + return memoryops.ListResult{}, fmt.Errorf("listMemories over http: the server returned %d memories with more remaining; "+ + "refusing a truncated answer (this client cannot page the memory plane)", len(page.Items)) + } // Empty map, never nil: a caller ranges over it without a nil check, and // "the plane is empty" is not something to spell as an absent answer. out := make(map[string]string, len(page.Items)) diff --git a/internal/httpclient/write_roles_test.go b/internal/httpclient/write_roles_test.go index a4456dab6..f4645ef0a 100644 --- a/internal/httpclient/write_roles_test.go +++ b/internal/httpclient/write_roles_test.go @@ -1802,6 +1802,24 @@ func TestMemoryListAnswersAnEmptyMapNeverNil(t *testing.T) { } } +// TestMemoryListRefusesATruncatedPage: ListResult has no cursor, so a page that +// reports more memories than it carries is refused rather than returned as the +// whole plane (`bd export` would drop the rest silently). +func TestMemoryListRefusesATruncatedPage(t *testing.T) { + w := &stubWire{page: &apigen.MemoriesPage{HasMore: true, Items: []apigen.Memory{{Key: "a", Value: "1"}}}} + memories, err := stubStore(t, w).Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + res, err := memories.List(t.Context(), memoryops.ListRequest{}) + if err == nil { + t.Fatalf("List of a has_more page = %+v, want a refusal", res) + } + if !strings.Contains(err.Error(), "listMemories") { + t.Errorf("refusal %q does not name listMemories", err) + } +} + // TestWriteAccessorsRefuseWithoutATransport: a build that registered the backend // without linking a wire client is a wiring fault, and it must be named rather // than reached as a nil dereference three frames into a role. diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index b4ca8d266..e23cdd54a 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "dependencyeditor.go", "edgereader.go", "errors.go", + "export.go", "gates.go", "graphcounter.go", "importer.go", @@ -57,6 +58,7 @@ go_test( srcs = [ "claimer_external_test.go", "current_test.go", + "export_test.go", "gates_test.go", "issueops_external_test.go", "molecule_internal_test.go", diff --git a/issueops/export.go b/issueops/export.go new file mode 100644 index 000000000..242509513 --- /dev/null +++ b/issueops/export.go @@ -0,0 +1,130 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/types" +) + +// ExportRequest describes which issues `bd export` writes. +type ExportRequest struct { + // All exports every row: every status, every type (infra, templates, + // gates), and both planes, wisps included. + All bool + // IncludeInfra admits the workspace's infrastructure types (agents, roles, + // messages by default) without lifting the other default exclusions. + IncludeInfra bool +} + +// ExportRow is one exported issue, fully hydrated: its labels, its stored +// outgoing dependency records and its comment bodies, plus the three +// cardinalities the export line carries. +type ExportRow struct { + Issue *types.Issue + DependencyCount int + DependentCount int + CommentCount int +} + +// ReadExport answers `bd export`'s read set by composing three role reads that +// every backend serves (a remote one as listIssues, listDependencies and +// getIssue): +// +// - Reader.List, unlimited, for the rows, their labels and their counts; +// - EdgeReader.ReadEdges, for every row's stored dependency records; +// - Reader.Get with IncludeComments, for the comment bodies of each row whose +// comment count is nonzero (a row with no comments costs no read). +// +// THE SELECTION IS EXPORT'S, not the listing's. Every status is exported +// (AllFlag), and gates are always exported. Without All, templates and the +// workspace's infra types are excluded (IncludeInfra admits the latter), and +// TRUE wisps are excluded while no-history rows, which live on the wisps plane +// with ephemeral unset, are kept (GH#3649). A listing cannot say "the wisps +// plane minus its ephemeral rows", so the plane is admitted and the ephemeral +// rows are dropped here. That is exact because the listing is unlimited: no +// row is dropped before a limit could have counted it. +// +// A FAILED READ IS AN ERROR. Export is a data-integrity path, so a refusal or +// a missing relation never degrades to a shorter export. An anchor the edge +// read reports missing (a row deleted between the reads) is an error too, and +// so is a listing page marked HasMore: the listing is unlimited, so a +// truncated page means the reader did not honor that. +func ReadExport(ctx context.Context, rd Reader, edges EdgeReader, req ExportRequest) ([]ExportRow, error) { + if rd == nil || edges == nil { + return nil, &ErrUnsupported{Op: "ReadExport", Backend: "nil"} + } + unlimited := 0 + list := ListRequest{ + AllFlag: true, + IncludeGates: true, + IncludeEphemeral: true, + Limit: &unlimited, + } + // --all is EXPORT's scope, spelled as export defines it (infra, + // templates, gates and wisps), not ListRequest.IncludeAllTypes: that + // union is the listing's "hide nothing" contract, which a remote reader + // refuses to restate, and export has always named its own four. + list.IncludeInfra = req.All || req.IncludeInfra + list.IncludeTemplates = req.All + page, err := rd.List(ctx, list) + if err != nil { + return nil, fmt.Errorf("export: listing issues: %w", err) + } + if page.HasMore { + return nil, errors.New("export: listing issues: the listing returned a truncated page; export must never truncate") + } + + rows := make([]ExportRow, 0, len(page.Items)) + ids := make([]string, 0, len(page.Items)) + for _, item := range page.Items { + if item == nil || item.Issue == nil { + continue + } + if !req.All && item.Ephemeral { + continue + } + rows = append(rows, ExportRow{ + Issue: item.Issue, + DependencyCount: item.DependencyCount, + DependentCount: item.DependentCount, + CommentCount: item.CommentCount, + }) + ids = append(ids, item.ID) + } + if len(rows) == 0 { + return rows, nil + } + + edgeResult, err := edges.ReadEdges(ctx, EdgeReadRequest{IDs: ids}) + if err != nil { + return nil, fmt.Errorf("export: reading dependencies: %w", err) + } + deps := make(map[string][]*types.Dependency, len(edgeResult.Anchors)) + for _, anchor := range edgeResult.Anchors { + if anchor.Missing { + return nil, fmt.Errorf("export: reading dependencies: %s is no longer in the store", anchor.ID) + } + if len(anchor.Edges) > 0 { + deps[anchor.ID] = anchor.Edges + } + } + + for i := range rows { + issue := rows[i].Issue + issue.Dependencies = deps[issue.ID] + issue.Comments = nil + if rows[i].CommentCount == 0 { + continue + } + detail, err := rd.Get(ctx, GetRequest{ID: issue.ID, IncludeComments: true}) + if err != nil { + return nil, fmt.Errorf("export: reading comments of %s: %w", issue.ID, err) + } + if detail != nil { + issue.Comments = detail.Comments + } + } + return rows, nil +} diff --git a/issueops/export_test.go b/issueops/export_test.go new file mode 100644 index 000000000..d6024b3ab --- /dev/null +++ b/issueops/export_test.go @@ -0,0 +1,160 @@ +package issueops_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +type exportReader struct { + page []*types.IssueWithCounts + hasMore bool + lists []issueops.ListRequest + gets []issueops.GetRequest + err error + getErr error +} + +func (r *exportReader) Ready(context.Context, issueops.ReadyRequest) (issueops.IssuePage, error) { + return issueops.IssuePage{}, errors.New("ready is not export's read") +} + +func (r *exportReader) List(_ context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + r.lists = append(r.lists, req) + if r.err != nil { + return issueops.IssuePage{}, r.err + } + return issueops.IssuePage{Items: r.page, HasMore: r.hasMore}, nil +} + +func (r *exportReader) Get(_ context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + r.gets = append(r.gets, req) + if r.getErr != nil { + return nil, r.getErr + } + return &issueops.IssueDetails{Comments: []*types.Comment{{IssueID: req.ID, Text: "c of " + req.ID}}}, nil +} + +type exportEdges struct { + calls []issueops.EdgeReadRequest + missing string +} + +func (e *exportEdges) ReadEdges(_ context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + e.calls = append(e.calls, req) + out := issueops.EdgeReadResult{Anchors: []issueops.AnchorEdges{}} + for _, id := range req.IDs { + a := issueops.AnchorEdges{ID: id, Edges: []*types.Dependency{}, Missing: id == e.missing} + if id == "x-1" { + a.Edges = []*types.Dependency{{IssueID: "x-1", DependsOnID: "x-2", Type: types.DepBlocks}} + } + out.Anchors = append(out.Anchors, a) + } + return out, nil +} + +func exportPage() []*types.IssueWithCounts { + return []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "x-1", Labels: []string{"l"}}, DependencyCount: 1, CommentCount: 2}, + {Issue: &types.Issue{ID: "x-2"}, DependentCount: 1}, + {Issue: &types.Issue{ID: "x-wisp", Ephemeral: true}}, + {Issue: &types.Issue{ID: "x-nh", NoHistory: true}}, + nil, + } +} + +// TestReadExportAllExportsEveryRowThroughRoles is S21: `bd export --all` +// composes served role reads, so a remote backend exports without the raw +// search and bulk relation reads it does not serve. +func TestReadExportAllExportsEveryRowThroughRoles(t *testing.T) { + rd := &exportReader{page: exportPage()} + edges := &exportEdges{} + rows, err := issueops.ReadExport(context.Background(), rd, edges, issueops.ExportRequest{All: true}) + if err != nil { + t.Fatalf("ReadExport: %v", err) + } + zero := 0 + wantList := issueops.ListRequest{AllFlag: true, IncludeGates: true, IncludeEphemeral: true, IncludeInfra: true, IncludeTemplates: true, Limit: &zero} + if len(rd.lists) != 1 || !reflect.DeepEqual(rd.lists[0], wantList) { + t.Fatalf("List calls = %#v, want one %#v", rd.lists, wantList) + } + var ids []string + for _, r := range rows { + ids = append(ids, r.Issue.ID) + } + if want := []string{"x-1", "x-2", "x-wisp", "x-nh"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("rows = %v, want %v (every row, in list order)", ids, want) + } + if len(edges.calls) != 1 || !reflect.DeepEqual(edges.calls[0].IDs, ids) { + t.Fatalf("ReadEdges calls = %#v, want one over %v", edges.calls, ids) + } + if got := rows[0].Issue.Dependencies; len(got) != 1 || got[0].DependsOnID != "x-2" { + t.Fatalf("x-1 dependencies = %#v, want the stored edge onto x-2", got) + } + if rows[1].Issue.Dependencies != nil { + t.Fatalf("x-2 dependencies = %#v, want none", rows[1].Issue.Dependencies) + } + // Only the row with comments is read for them. + if want := []issueops.GetRequest{{ID: "x-1", IncludeComments: true}}; !reflect.DeepEqual(rd.gets, want) { + t.Fatalf("Get calls = %#v, want %#v", rd.gets, want) + } + if c := rows[0].Issue.Comments; len(c) != 1 || c[0].Text != "c of x-1" { + t.Fatalf("x-1 comments = %#v", c) + } + if rows[0].CommentCount != 2 || rows[0].DependencyCount != 1 || rows[1].DependentCount != 1 { + t.Fatalf("counts lost: %+v %+v", rows[0], rows[1]) + } + if !reflect.DeepEqual(rows[0].Issue.Labels, []string{"l"}) { + t.Fatalf("labels lost: %v", rows[0].Issue.Labels) + } +} + +// TestReadExportDefaultDropsTrueWispsKeepsNoHistory pins the default scope: +// gates and the wisps plane are read, templates and infra are the listing's to +// exclude, and the true wisps are dropped while no-history rows stay (GH#3649). +func TestReadExportDefaultDropsTrueWispsKeepsNoHistory(t *testing.T) { + for _, infra := range []bool{false, true} { + rd := &exportReader{page: exportPage()} + rows, err := issueops.ReadExport(context.Background(), rd, &exportEdges{}, issueops.ExportRequest{IncludeInfra: infra}) + if err != nil { + t.Fatalf("ReadExport: %v", err) + } + zero := 0 + wantList := issueops.ListRequest{AllFlag: true, IncludeGates: true, IncludeEphemeral: true, IncludeInfra: infra, Limit: &zero} + if !reflect.DeepEqual(rd.lists[0], wantList) { + t.Fatalf("List = %#v, want %#v", rd.lists[0], wantList) + } + var ids []string + for _, r := range rows { + ids = append(ids, r.Issue.ID) + } + if want := []string{"x-1", "x-2", "x-nh"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("rows = %v, want %v", ids, want) + } + } +} + +func TestReadExportNeverShortensOnAFailure(t *testing.T) { + refusal := &issueops.ErrUnsupported{Op: "Reader.List", Backend: "http"} + if rows, err := issueops.ReadExport(context.Background(), &exportReader{err: refusal}, &exportEdges{}, issueops.ExportRequest{All: true}); !errors.Is(err, refusal) || rows != nil { + t.Fatalf("ReadExport = (%v, %v), want (nil, the refusal)", rows, err) + } + if _, err := issueops.ReadExport(context.Background(), &exportReader{page: exportPage()}, &exportEdges{missing: "x-2"}, issueops.ExportRequest{All: true}); err == nil { + t.Fatal("ReadExport with a vanished anchor succeeded, want an error") + } + if rows, err := issueops.ReadExport(context.Background(), &exportReader{page: exportPage(), hasMore: true}, &exportEdges{}, issueops.ExportRequest{All: true}); err == nil || rows != nil { + t.Fatalf("ReadExport over a truncated listing = (%v, %v), want (nil, an error)", rows, err) + } + notFound := errors.New("x-1 not found") + if rows, err := issueops.ReadExport(context.Background(), &exportReader{page: exportPage(), getErr: notFound}, &exportEdges{}, issueops.ExportRequest{All: true}); !errors.Is(err, notFound) || rows != nil { + t.Fatalf("ReadExport with a failed comment read = (%v, %v), want (nil, the failure)", rows, err) + } + var unsupported *issueops.ErrUnsupported + if _, err := issueops.ReadExport(context.Background(), nil, nil, issueops.ExportRequest{}); !errors.As(err, &unsupported) { + t.Fatalf("ReadExport(nil) = %v, want ErrUnsupported", err) + } +} From 048f5e5192c3f104d915c6f51e08cde1666361e6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 14 Oct 2026 05:04:19 +0000 Subject: [PATCH 05/76] fix(httpclient): bound response-header waits and retry stalled reads on another connection (#7553) * fix(httpclient): bound response-header waits and retry stalled reads on a new connection The maintainer-city shadow run saw 22 requests sit out the full 60s client timeout, each on one silent HTTP/1.1 connection while other connections to the same serve kept answering. wire.Client now bounds an idempotent request's wait for response headers (15s default; BEADS_HTTP_RESPONSE_HEADER_TIMEOUT or DialOptions.ResponseHeaderTimeout; "off" disables), separately from the 60s overall timeout, and stops the timer once headers arrive so a streaming body is never cut. A read (GET/HEAD/OPTIONS, or a Request marked Idempotent: the POST batch get) that stalls or has its connection reset is retried once: the stalled attempt's cancellation closes its connection and idle connections are dropped, so the retry dials fresh. Every retry is counted (wire.StallRetryCount) and reported (OnStallRetry; the dialer's default logs one stderr line with the count). Writes are never retried and get no header timer, because bd serve answers a write only after it commits (a large batch apply can legitimately take tens of seconds). A write whose overall timeout fires before its headers fails with a typed *wire.ResponseHeaderTimeoutError (inside *ConnectError) naming the host and saying the outcome is unknown. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): retry a stalled read within the overall timeout (bd-p1wzpi) Adoption review of #7553 found three defects in the header timer and its retry. This maintainer fixup sits on top of the contributor commit and supersedes three of its claims: that the timer bounds a read's every attempt, that idle connections are dropped before the retry, and that any write whose overall timeout fires before its headers is typed. A slow but healthy read is no longer cut short. The 15s timer cannot tell a silent connection from a serve still working on the read (serve writes a read's headers only after waiting up to 10s for a database slot and running the query), and the retry ran under a fresh 15s timer, so a read slower than 15s failed at about 30s after sending two requests where it used to succeed within 60s. The timer now bounds only the first attempt. The retry has no header timer and runs under a deadline of when the first attempt was sent plus the client's overall Timeout, the moment that Timeout itself counts from, so it finishes such a read and no read waits longer than the 60s it could before. The request is built and stamped once and each attempt sends a copy of it, so the credential source is consulted once, before any clock starts, and a cold token command costs the retry none of its budget. What is left: a read whose headers take longer than the timer, and whose retry then needs more than what remains of the 60s (about 45s) for its headers and body together, now fails at 60s where it used to succeed, and the server also gets the abandoned first request. A read whose first attempt already spent the whole Timeout (the timer off) is not retried at all and keeps its typed timeout. A request is typed *ResponseHeaderTimeoutError only once it was written. Any Timeout() error on a write used to be typed, including the baseline transport's dial and TLS handshake timeouts where nothing reached the server, and the message then said the write "may or may not have been applied", gave the overall timeout as the wait, dropped the transport's error, and Temporary() was true. The timer is now armed from httptrace's WroteRequest (where http.Transport.ResponseHeaderTimeout starts its own clock), which also records that the request went out. A timeout is typed only for a written request whose caller's context is still alive; a dial or TLS timeout stays a plain *ConnectError. After is the measured wait, Err is printed, Temporary() reports Replayable (false for a write), and a retried read says why its first attempt was abandoned (Reason). Arming at the write also stops a slow credential source (BEADS_HTTP_TOKEN_COMMAND may take 30s) from being reported, counted and retried as a server stall. No idle-connection flush. CloseIdleConnections ran on a transport shared by every no-CA dial in the process (baselineTransport), so one stalled read dropped idle keep-alives to every host. Canceling the stalled attempt already closes its connection (net/http never pools a canceled HTTP/1.1 connection), so the retry rides another one. Also: - wire.Request.Replayable (GET/HEAD/OPTIONS, or Idempotent) is now the one predicate behind both the wire's stall retry and dispatch's bounded 503 retry, so the batch get, a read on POST, is retried on a 503 too. dispatch's doc comment no longer says the wire never resends. - The docs, the error and the stall line say the retry goes out on "another connection", not a "new" one: the idle pool may hand it one already open. dialStream's doc says the events:watch connect gets neither the timer nor the retry. - BEADS_HTTP_RESPONSE_HEADER_TIMEOUT takes bare seconds ("30") as BEADS_GATE_WAIT_TIMEOUT does, and "off" in any case. "off" or 0 turns off only the timer: a reset connection is still retried and a written request that times out is still typed. - README says "each read" and spells out the above, bounding the retry by its headers and body together; it no longer says a streaming body is never cut off (the header timer never does, the 60s bound still does). The CHANGELOG entry moves from Added to Fixed and states the same bound. - CredentialProvider's doc, in wire and on the public bdhttp alias, says Authorize runs once per request, not once per attempt: the stall retry resends the stamped headers up to the overall timeout later, so a single-use or short-lived credential may draw a 401 there, which reaches Refresh like any other. Tests: a read queued behind a LimitListener backlog finishes once a slot frees; a read slower than the timer on every connection finishes on its retry; a slow credential source is not a stall, and one that takes the whole Timeout to resolve still leaves the retry its budget (starting the deadline before the stamp, or re-stamping the retry, fails it); a write whose dial times out is a plain *ConnectError; a stall leaves another server's idle connection on a shared transport alone; a retry after a reset names the reset; the stall-twice case is bounded by the overall timeout and pins After, Reason and the message; dispatch retries a read on POST on a 503, and GetMany's batch get is sent as one (dropping its Idempotent fails that); the env knob takes "30", " 1.5 " and "OFF" and refuses "-3". A read whose first attempt spent the whole Timeout is not retried and keeps its typed error (dropping the spent-budget check fails it; no other test did). A stall retry that draws a 401 refreshes once and authorizes only the re-issued request, so Authorize runs twice, not three times (re-stamping the retry fails it). Not covered, as before, and now said in README and the CHANGELOG: the events:watch stream's connect (dialStream) has no header timer or retry. Nor is a reset on Windows: stallReason matches syscall.ECONNRESET, which a Windows reset (WSAECONNRESET) is not, so there a reset connection is retried only when it surfaces as EOF. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Eddie the Engineer --- CHANGELOG.md | 15 + README.md | 23 + backend/http/credential.go | 6 + internal/httpclient/BUILD.bazel | 1 + internal/httpclient/batchgetter.go | 2 + internal/httpclient/dial.go | 70 +- .../httpclient/dial_header_timeout_test.go | 97 +++ internal/httpclient/dispatch.go | 22 +- internal/httpclient/dispatch_test.go | 48 ++ internal/httpclient/wire/BUILD.bazel | 3 + internal/httpclient/wire/client.go | 160 +++- internal/httpclient/wire/credential.go | 7 + internal/httpclient/wire/stall_retry.go | 261 ++++++ internal/httpclient/wire/stall_retry_test.go | 775 ++++++++++++++++++ internal/httpclient/wire/stream.go | 8 + 15 files changed, 1480 insertions(+), 18 deletions(-) create mode 100644 internal/httpclient/dial_header_timeout_test.go create mode 100644 internal/httpclient/wire/stall_retry.go create mode 100644 internal/httpclient/wire/stall_retry_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 580099cca..73857d206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -476,6 +476,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 of a remote export puts genuine no-history wisps on the durable issues plane instead of the wisps plane, and gc's JSONL archive keeps them. Nothing is dropped. +- **A silent HTTP connection no longer costs a read the full 60s.** The HTTP + backend client gives a read's first attempt 15s from when it is sent to get + its response headers (`BEADS_HTTP_RESPONSE_HEADER_TIMEOUT`: a duration, a + number of seconds, or `off`), then retries it once on another connection + with no header timer, on what is left of the same 60s request timeout. A + slow but healthy read still finishes if its retry's headers and body + together fit in that remainder (about 45s); one that needs longer now fails + at 60s. A read whose connection is reset is retried the same way (on Windows + only when the reset surfaces as an early EOF), and each retry is logged to + stderr with its count. The events:watch stream's connect gets neither the + timer nor the retry. A write is never retried; one that was sent and timed + out before its headers fails with a typed `wire.ResponseHeaderTimeoutError` + naming the server's host, because it may or may not have been applied, + while a dial or TLS timeout stays a plain connection error. The batch get, + a read on POST, is now also retried on a 503 the way a GET is. - **An update of a template is now refused for every caller, not only `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item diff --git a/README.md b/README.md index 8f16f1a28..570bdc01c 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,29 @@ credential would cross the network unencrypted) — pass `--allow-plaintext` to override, or use `https://`. Switching a workspace that already selects a different backend needs `--force`. +Each read (a GET, or the POST batch get) gives its first attempt 15s, from +when the request is sent, to get the server's response headers +(`BEADS_HTTP_RESPONSE_HEADER_TIMEOUT`: a duration such as `30s`, a number of +seconds, or `off`). A read whose headers do not arrive in time, or whose +connection is reset, is retried once on another connection, and each retry +is logged to stderr with its count. The retry has no header timer, but it +runs on what is left of the 60s bound on the whole request, counted from +when the first attempt was sent: a slow but healthy read still finishes if +the retry's headers and body together fit in that remainder (about 45s after +a header timeout), and one that needs longer now fails at 60s. The header +timer never cuts off a body that is still streaming; the 60s bound still +does. The events:watch stream's connect gets neither the header timer nor +the retry. On Windows a reset connection is retried only when it surfaces as +an early EOF, because the check matches the Unix reset error (`ECONNRESET`), +which a Windows reset (`WSAECONNRESET`) is not. A write gets no +header timer (the server answers a write only after it commits, so a large +one can be slow and healthy) and is never retried; if the 60s bound expires +after it was sent but before its headers, it fails with an error naming the +server's host, because it may or may not have been applied. A dial or TLS +timeout stays a plain connection error, since the server never saw the +request. `off` turns off only the header timer: a reset read is still +retried, and a write's timeout is still reported as above. + `bd search`, `bd show --current` and `bd recompute-blocked` work over HTTP too, each in its own way. `bd search` asks the server's listing with its text (`q`, behind the `issues.list.search` capability) and refuses before dialing against a server that does not advertise it. `bd list diff --git a/backend/http/credential.go b/backend/http/credential.go index ffd46987e..3290fd3b7 100644 --- a/backend/http/credential.go +++ b/backend/http/credential.go @@ -12,6 +12,12 @@ import ( // persist a secret, and must FAIL CLOSED: a configured source that errors // aborts the request rather than silently downgrading to an unauthenticated // one. +// +// Authorize is called once per request, not once per attempt: a read whose +// first attempt stalls or has its connection reset is resent once with the +// same headers, up to the client's overall timeout later. A provider whose +// credentials are single-use or short-lived should expect that resend to draw +// a 401, which reaches Refresh like any other. type CredentialProvider = httpclient.CredentialProvider const ( diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 611b53d50..6beb6b8ec 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -107,6 +107,7 @@ go_test( "create_priority_test.go", "credential_source_test.go", "credential_test.go", + "dial_header_timeout_test.go", "dial_test.go", "dispatch_test.go", "divergence_citation_gate_test.go", diff --git a/internal/httpclient/batchgetter.go b/internal/httpclient/batchgetter.go index c1f6eccd3..db12906fb 100644 --- a/internal/httpclient/batchgetter.go +++ b/internal/httpclient/batchgetter.go @@ -84,6 +84,8 @@ func (g *httpBatchGetter) GetMany(ctx context.Context, req issueops.GetManyReque Body: apigen.BatchGetIssuesRequest{ Ids: append([]string(nil), req.IDs...), }, + // A read on POST: safe to replay after a stall or a 503, as a GET is. + Idempotent: true, }, &body); err != nil { return issueops.GetManyResult{}, err } diff --git a/internal/httpclient/dial.go b/internal/httpclient/dial.go index dcfadb217..5a33ef1bb 100644 --- a/internal/httpclient/dial.go +++ b/internal/httpclient/dial.go @@ -7,8 +7,10 @@ import ( "errors" "fmt" "net/http" + "os" "strings" "sync" + "time" "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/wire" @@ -94,6 +96,58 @@ type DialOptions struct { // Unexported so the only way to set it is the door that also refuses a // nil credential. explicitCredential bool + + // ResponseHeaderTimeout bounds a read's first attempt's wait for response + // headers, separately from the client's overall timeout + // (wire.DefaultTimeout): see wire.Options.ResponseHeaderTimeout. Zero takes + // ResponseHeaderTimeoutEnv, else wire.DefaultResponseHeaderTimeout; + // negative disables it. + ResponseHeaderTimeout time.Duration + // OnStallRetry is told about every retry of a stalled or reset idempotent + // request. Nil logs one line per retry to stderr, so a stall is visible + // rather than silently absorbed. + OnStallRetry func(wire.StallRetry) +} + +// ResponseHeaderTimeoutEnv overrides the header timeout for every dial that +// does not set DialOptions.ResponseHeaderTimeout: a Go duration ("15s", +// "2m") or a number of seconds ("30"), as BEADS_GATE_WAIT_TIMEOUT takes, or "0"/ +// "off" to disable it. Disabling it turns off only the timer: a read whose +// connection is reset is still retried once, and a write that times out +// awaiting its headers is still reported as one. An unparseable value +// refuses the dial. +const ResponseHeaderTimeoutEnv = "BEADS_HTTP_RESPONSE_HEADER_TIMEOUT" + +// responseHeaderTimeout resolves the header timeout for one dial: the +// explicit option, else ResponseHeaderTimeoutEnv, else 0 (wire's default). +func responseHeaderTimeout(opt time.Duration) (time.Duration, error) { + if opt != 0 { + return opt, nil + } + raw := strings.TrimSpace(os.Getenv(ResponseHeaderTimeoutEnv)) + if raw == "" { + return 0, nil + } + if strings.EqualFold(raw, "off") { + return -1, nil + } + d, err := time.ParseDuration(raw) + if err != nil { + d, err = time.ParseDuration(raw + "s") + } + if err != nil || d < 0 { + return 0, fmt.Errorf("%s=%q: want a non-negative duration such as 15s or 30, or off", ResponseHeaderTimeoutEnv, raw) + } + if d == 0 { + return -1, nil + } + return d, nil +} + +// logStallRetry is the default OnStallRetry: one stderr line per retry, with +// the retry number and the process-wide count. +func logStallRetry(e wire.StallRetry) { + fmt.Fprintln(os.Stderr, e.String()) } // Conn is the store's view of one wire client. @@ -226,10 +280,20 @@ func DialWith(target Target, creds CredentialProvider, opts DialOptions) (*Conn, // SAME flag before the sidecar carrying it even exists yet. creds = guardInsecureCredential(target, creds, opts.AllowInsecureCredential || target.AllowInsecureCredential, !opts.explicitCredential) + headerTimeout, err := responseHeaderTimeout(opts.ResponseHeaderTimeout) + if err != nil { + return nil, err + } + onStall := opts.OnStallRetry + if onStall == nil { + onStall = logStallRetry + } client, err := wire.New(target.BaseURL, creds, wire.Options{ - HTTPClient: opts.HTTPClient, - UserAgent: opts.UserAgent, - ExpectProjectID: target.ExpectProjectID, + HTTPClient: opts.HTTPClient, + UserAgent: opts.UserAgent, + ExpectProjectID: target.ExpectProjectID, + ResponseHeaderTimeout: headerTimeout, + OnStallRetry: onStall, }) if err != nil { return nil, err diff --git a/internal/httpclient/dial_header_timeout_test.go b/internal/httpclient/dial_header_timeout_test.go new file mode 100644 index 000000000..98620205f --- /dev/null +++ b/internal/httpclient/dial_header_timeout_test.go @@ -0,0 +1,97 @@ +package httpclient + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +func TestResponseHeaderTimeoutResolution(t *testing.T) { + cases := []struct { + opt time.Duration + env string + want time.Duration + wantErr bool + }{ + {opt: 0, env: "", want: 0}, // wire's default + {opt: 0, env: "5s", want: 5 * time.Second}, + {opt: 0, env: "30", want: 30 * time.Second}, // seconds, as BEADS_GATE_WAIT_TIMEOUT takes + {opt: 0, env: " 1.5 ", want: 1500 * time.Millisecond}, + {opt: 0, env: "off", want: -1}, + {opt: 0, env: "OFF", want: -1}, + {opt: 0, env: "0", want: -1}, + {opt: 0, env: "0s", want: -1}, + {opt: 0, env: "soon", wantErr: true}, + {opt: 0, env: "-3s", wantErr: true}, + {opt: 0, env: "-3", wantErr: true}, + {opt: 2 * time.Second, env: "5s", want: 2 * time.Second}, // the option wins + } + for _, tc := range cases { + t.Setenv(ResponseHeaderTimeoutEnv, tc.env) + got, err := responseHeaderTimeout(tc.opt) + if (err != nil) != tc.wantErr { + t.Errorf("opt=%v env=%q: err = %v, wantErr %v", tc.opt, tc.env, err, tc.wantErr) + continue + } + if !tc.wantErr && got != tc.want { + t.Errorf("opt=%v env=%q: got %v, want %v", tc.opt, tc.env, got, tc.want) + } + } +} + +func TestAnUnparseableHeaderTimeoutEnvRefusesTheDial(t *testing.T) { + clearCAEnvironment(t) + t.Setenv(ResponseHeaderTimeoutEnv, "soon") + if _, err := Dial(Target{BaseURL: mustParseURL(t, "http://127.0.0.1:1")}, DialOptions{}); err == nil { + t.Fatal("Dial succeeded with an unparseable header timeout") + } +} + +// TestTheProductionDialRetriesAStalledReadOnAnotherConnection is the shadow +// run's stall end to end through the production transport DialWith builds: +// the first connection never answers, the read is retried once on another one +// and succeeds within about the header timeout, not the 60s overall timeout. +func TestTheProductionDialRetriesAStalledReadOnAnotherConnection(t *testing.T) { + clearCAEnvironment(t) + t.Setenv(ResponseHeaderTimeoutEnv, "300ms") + var mu sync.Mutex + first := "" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + if first == "" { + first = r.RemoteAddr + } + stuck := r.RemoteAddr == first + mu.Unlock() + if stuck { + <-r.Context().Done() + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) + })) + t.Cleanup(srv.Close) + + var events []wire.StallRetry + conn, err := Dial(Target{BaseURL: mustParseURL(t, srv.URL)}, DialOptions{ + OnStallRetry: func(e wire.StallRetry) { events = append(events, e) }, + }) + if err != nil { + t.Fatalf("Dial: %v", err) + } + start := time.Now() + if err := conn.Health(context.Background()); err != nil { + t.Fatalf("Health: %v", err) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("elapsed %v, want about one 300ms header timeout", elapsed) + } + if len(events) != 1 || events[0].Retry != 1 { + t.Fatalf("events = %+v, want exactly one retry", events) + } +} diff --git a/internal/httpclient/dispatch.go b/internal/httpclient/dispatch.go index a168668d8..eefc145e2 100644 --- a/internal/httpclient/dispatch.go +++ b/internal/httpclient/dispatch.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "net/http" "time" "github.com/steveyegge/beads/internal/httpclient/wire" @@ -37,10 +36,12 @@ const ( // of a reviewer's memory: a role that wants to reach the server has no other // route to it. // -// It is also the one place that knows an operation's HTTP method, so the bounded -// 503 retry lives here: the wire layer bounds Retry-After and hands it back but -// never sleeps or resends, precisely because whether a 503 is safe to come back -// to is a question only the method answers, and only a GET's answer is yes. +// The bounded 503 retry lives here too: the wire layer bounds Retry-After and +// hands it back but never sleeps on it. Whether a 503 is safe to come back to +// is the question wire.Request.Replayable answers, and only a read's answer +// is yes. (The wire does resend a replayable request once, at once, when its +// first attempt stalled before any headers or had its connection reset; that +// retry answers a transport-level failure, a 503 is the server's answer.) // // A store with no transport is a build-wiring fault (ErrNoTransport), not the // user's: the workspace and the server may both be fine. @@ -58,11 +59,12 @@ func (s *Store) dispatch(ctx context.Context, req wire.Request, out any) error { } err := s.wire.Do(ctx, req, out) - // Only an idempotent GET is retried. A 503 can interrupt a non-idempotent - // write whose first attempt already committed before the server ran out of - // slots, so resending a POST/PATCH/DELETE could double-apply it — DELETE - // included, since forgetMemory is a mutation the wire happens to spell as one. - if req.Method != http.MethodGet { + // Only a replayable read is retried: a GET, or a read on POST that says so + // (the batch get). A 503 can interrupt a non-idempotent write whose first + // attempt already committed before the server ran out of slots, so + // resending a POST/PATCH/DELETE could double-apply it — DELETE included, + // since forgetMemory is a mutation the wire happens to spell as one. + if !req.Replayable() { return err } for attempt := 0; attempt < maxIdempotentRetries; attempt++ { diff --git a/internal/httpclient/dispatch_test.go b/internal/httpclient/dispatch_test.go index a297b5acb..624ae32e9 100644 --- a/internal/httpclient/dispatch_test.go +++ b/internal/httpclient/dispatch_test.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" ) // busyThenOK answers with a 503 for the first busyCount Do calls and succeeds @@ -79,6 +80,53 @@ func TestDispatchCapsIdempotentRetries(t *testing.T) { } } +// TestDispatchRetriesAReadOnPostOnBusy: a read carried on POST that marks +// itself Idempotent comes back after a 503 the way a GET does. That the batch +// get marks itself is TestGetManyRetriesTheBatchGetOnBusy's. +func TestDispatchRetriesAReadOnPostOnBusy(t *testing.T) { + server := &busyThenOK{busyCount: 1, retryAfter: time.Millisecond} + s := retryTestStore(t, server.do) + + read := wire.Request{Op: wire.OpBatchGetIssues, Method: http.MethodPost, Path: wire.PathIssuesBatchGet, Idempotent: true} + if err := s.dispatch(context.Background(), read, nil); err != nil { + t.Fatalf("dispatch: %v, want the retry to succeed", err) + } + if server.calls != 2 { + t.Errorf("Do called %d times, want 2 (one dial plus one retry)", server.calls) + } +} + +// TestGetManyRetriesTheBatchGetOnBusy: the batch get is a read dialed as a +// POST, so it is replayable only because it says so (Idempotent). Through +// GetMany it comes back after a 503, and what it sends is replayable for the +// wire's stall retry too. +func TestGetManyRetriesTheBatchGetOnBusy(t *testing.T) { + token, _ := wire.CapabilityFor(wire.OpBatchGetIssues) + server := &busyThenOK{busyCount: 1, retryAfter: time.Millisecond} + var sent []wire.Request + w := &fakeWire{ + res: &apigen.ContextResponse{Capabilities: []string{token}}, + preflight: func(context.Context, string) error { return nil }, + do: func(ctx context.Context, req wire.Request, out any) error { + sent = append(sent, req) + return server.do(ctx, req, out) + }, + } + g := &httpBatchGetter{store: New(testTarget(t), w, nil)} + + if _, err := g.GetMany(context.Background(), issueops.GetManyRequest{IDs: []string{"bd-1"}}); err != nil { + t.Fatalf("GetMany: %v, want the busy batch get retried", err) + } + if server.calls != 2 { + t.Errorf("Do called %d times, want 2 (one dial plus one retry)", server.calls) + } + for _, req := range sent { + if req.Op != wire.OpBatchGetIssues || req.Method != http.MethodPost || !req.Replayable() { + t.Errorf("sent %s %s (replayable %v), want the batch get as a replayable POST", req.Method, req.Op, req.Replayable()) + } + } +} + // TestDispatchDoesNotRetryWrites: a non-idempotent write is dialed exactly once // on a 503, because its first attempt may have committed before the server ran // out of slots and a resend would double-apply it. diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel index 012a18995..53dc2522b 100644 --- a/internal/httpclient/wire/BUILD.bazel +++ b/internal/httpclient/wire/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "paths.go", "problem.go", "revision_tolerance.go", + "stall_retry.go", "stream.go", "writes.go", ], @@ -37,6 +38,7 @@ go_test( "project_stamp_test.go", "revision_tolerance_test.go", "skew_test.go", + "stall_retry_test.go", "stream_test.go", "vocabulary_test.go", "writes_batch_cap_test.go", @@ -57,6 +59,7 @@ go_test( "//internal/types", "//issueops", "@in_gopkg_yaml_v3//:yaml_v3", + "@org_golang_x_net//netutil", ], ) diff --git a/internal/httpclient/wire/client.go b/internal/httpclient/wire/client.go index 76955f51f..90c8beb9c 100644 --- a/internal/httpclient/wire/client.go +++ b/internal/httpclient/wire/client.go @@ -73,6 +73,22 @@ type Options struct { // handshake compares it against ContextResponse.project_id; "" skips the // check. ExpectProjectID string + // ResponseHeaderTimeout bounds the first attempt's wait for response + // headers on an idempotent request (GET, or Request.Idempotent), counted + // from when the request has been written and separately from the + // HTTPClient's overall Timeout: zero means DefaultResponseHeaderTimeout, + // negative disables it. A body that is still streaming once the headers + // arrived is never cut off by it. Such a request that hits it, or whose + // connection is reset, is retried once on another connection, with no + // header timer and only what is left of the overall Timeout, counted from + // when the first attempt was sent. A write is never + // retried and is bounded by the overall Timeout alone (bd serve answers a + // write after it commits). A written request whose headers do not come + // back in time fails with *ResponseHeaderTimeoutError. + ResponseHeaderTimeout time.Duration + // OnStallRetry, when set, is told about every such retry. The process-wide + // count is StallRetryCount either way. + OnStallRetry func(StallRetry) } // Client is one workspace's connection to one bd serve. @@ -87,6 +103,9 @@ type Client struct { maxBytes int64 maxRetry time.Duration + headerTimeout time.Duration + onStall func(StallRetry) + handshake handshakeCache expectID string } @@ -124,6 +143,13 @@ func New(base *url.URL, creds CredentialProvider, opts Options) (*Client, error) maxBytes: opts.MaxResponseBytes, maxRetry: opts.MaxRetryAfter, expectID: opts.ExpectProjectID, + onStall: opts.OnStallRetry, + } + switch { + case opts.ResponseHeaderTimeout == 0: + c.headerTimeout = DefaultResponseHeaderTimeout + case opts.ResponseHeaderTimeout > 0: + c.headerTimeout = opts.ResponseHeaderTimeout } if c.userAgent == "" { c.userAgent = DefaultUserAgent @@ -212,6 +238,23 @@ type Request struct { // error names it — the close's not_assignee — when the wire does not // repeat what the request already said. Actor string + // Idempotent marks a read carried on a non-GET method (a POST batch get) + // as safe to replay, the way a GET is (see Replayable). Never set it on a + // write. + Idempotent bool +} + +// Replayable reports whether r may be sent again after an attempt that got no +// usable answer: the safe methods, or a read carried on POST that says so +// (Idempotent). It is the one predicate both retries consult: the wire's +// retry of a stalled or reset attempt on another connection, and the store's +// bounded retry of a 503. +func (r Request) Replayable() bool { + switch r.Method { + case http.MethodGet, http.MethodHead, http.MethodOptions: + return true + } + return r.Idempotent } // Do issues r and decodes a 2xx body into out, which may be nil for a response @@ -335,13 +378,70 @@ type response struct { body []byte } -// roundTrip issues one HTTP request and reads its whole bounded body. +// roundTrip issues r, retrying an idempotent request once on another +// connection when the first attempt stalls before its headers or has its +// connection reset (see Options.ResponseHeaderTimeout). // -// Reading the body here rather than streaming it to the caller is what makes -// the 401 retry above expressible at all: both attempts are then symmetric, and -// the request body is a []byte that can be replayed. The size cap is what makes -// that safe. +// The request is built and stamped once and each attempt sends a copy, so the +// retry replays what the first attempt sent and the credential source is +// consulted before any clock starts. The retry gets no header timer, and both +// attempts share the one overall budget the client's Timeout gives a request, +// counted, as that Timeout is, from when the first attempt is sent: a +// credential source slow to resolve (a cold token command) is not the +// server's time. The timer cannot tell a connection that went silent from a +// serve still working on a slow read, so a retry cut short by it again would +// fail a healthy read the client used to wait out. Bounded by what is left of +// the overall Timeout instead, such a read still finishes if its headers and +// body fit in what is left, and no read waits longer than that Timeout. func (c *Client) roundTrip(ctx context.Context, r Request, u *url.URL, body []byte) (*response, error) { + req, err := c.newRequest(ctx, r, u, body) + if err != nil { + return nil, err + } + if !r.Replayable() { + // Only a replayable request gets the header timer. bd serve sends a + // write's headers after the write commits, so a slow but healthy write + // (a large batch apply) can legitimately take longer than the header + // timeout, and cutting it would turn it into an unknown outcome that + // cannot be retried. A write stays bounded by the client's overall + // timeout, and a timeout awaiting its headers is typed in + // roundTripOnce. + return c.roundTripOnce(ctx, r, u, req, 0, time.Time{}) + } + sent := time.Now() + res, err := c.roundTripOnce(ctx, r, u, req, c.headerTimeout, time.Time{}) + if err == nil { + return res, nil + } + var deadline time.Time + if c.hc.Timeout > 0 { + deadline = sent.Add(c.hc.Timeout) + } + reason := stallReason(err) + if reason == "" || ctx.Err() != nil || (!deadline.IsZero() && !time.Now().Before(deadline)) { + return nil, err + } + // No idle-pool flush: canceling the stalled attempt closed its connection + // (net/http never pools a canceled HTTP/1.1 connection, which is what bd + // serve speaks; behind an HTTP/2 proxy the retry may share the proxy's + // connection), so the retry rides another one. The transport may also be + // shared by every client in the process, and its idle connections to + // other servers are none of this request's business. + event := StallRetry{Op: r.Op, Method: r.Method, Host: u.Host, Reason: reason, Retry: 1, Total: stallRetries.Add(1), Err: err} + if c.onStall != nil { + c.onStall(event) + } + res, err = c.roundTripOnce(ctx, r, u, req, 0, deadline) + var hte *ResponseHeaderTimeoutError + if errors.As(err, &hte) { + hte.After, hte.Attempts, hte.Retried, hte.Reason = time.Since(sent), 2, true, reason + } + return res, err +} + +// newRequest builds r's HTTP request and stamps it (see stampRequest). +// roundTripOnce sends a copy of it per attempt. +func (c *Client) newRequest(ctx context.Context, r Request, u *url.URL, body []byte) (*http.Request, error) { var rdr io.Reader if body != nil { rdr = bytes.NewReader(body) @@ -357,10 +457,60 @@ func (c *Client) roundTrip(ctx context.Context, r Request, u *url.URL, body []by if err := c.stampRequest(ctx, r.Op, req); err != nil { return nil, err } + return req, nil +} + +// roundTripOnce sends one copy of the stamped request and reads its whole +// bounded body. headerTimeout (0: none) bounds its wait for headers once it is +// written, and deadline (zero: none) bounds the whole attempt on top of ctx +// and the client's own Timeout. +// +// Reading the body here rather than streaming it to the caller is what makes +// the 401 retry above expressible at all: both attempts are then symmetric, and +// the request body is a []byte that can be replayed. The size cap is what makes +// that safe. +func (c *Client) roundTripOnce(ctx context.Context, r Request, u *url.URL, stamped *http.Request, headerTimeout time.Duration, deadline time.Time) (*response, error) { + attemptCtx := ctx + if !deadline.IsZero() { + var cancel context.CancelFunc + attemptCtx, cancel = context.WithDeadline(ctx, deadline) + defer cancel() + } + attemptCtx, ht := startHeaderTimer(attemptCtx, headerTimeout) + defer ht.done() + req := stamped.Clone(attemptCtx) + if stamped.GetBody != nil { + // Clone shares the body reader, which an earlier attempt may have + // consumed; every attempt sends the body from its start. + b, err := stamped.GetBody() + if err != nil { + return nil, fmt.Errorf("%s: %w", r.Op, err) + } + req.Body = b + } //nolint:gosec // G704: the URL is the workspace's configured server, resolved from the sidecar or --server-url; dialing it IS the feature. resp, err := c.hc.Do(req) + if ht.stop() { + // The header timer fired first. Headers that raced it are discarded: + // the attempt's context is already canceled, so its body is lost. + if resp != nil { + _ = resp.Body.Close() + } + hte := &ResponseHeaderTimeoutError{Op: r.Op, Method: r.Method, Host: u.Host, After: headerTimeout, Attempts: 1, Replayable: r.Replayable()} + return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: hte} + } if err != nil { + // A timeout after the request was written ran out awaiting its + // headers: Do returns once they arrive, so whichever bound fired (the + // client's Timeout or this attempt's deadline) fired before them. A + // timeout before the request was written (a dial or TLS handshake) + // stays a plain ConnectError, because the server cannot have seen + // the request. The caller's own context expiring is the caller's. + if wroteAt, ok := ht.written(); ok && ctx.Err() == nil && isTimeout(err) { + hte := &ResponseHeaderTimeoutError{Op: r.Op, Method: r.Method, Host: u.Host, After: time.Since(wroteAt), Attempts: 1, Replayable: r.Replayable(), Err: err} + return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: hte} + } return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: err} } defer func() { _ = resp.Body.Close() }() diff --git a/internal/httpclient/wire/credential.go b/internal/httpclient/wire/credential.go index f8a2224ee..96b15228d 100644 --- a/internal/httpclient/wire/credential.go +++ b/internal/httpclient/wire/credential.go @@ -25,6 +25,13 @@ type CredentialProvider interface { // A no-auth deployment — the loopback-trust OSS server, which has no token // file at all — returns nil without modifying req. // + // It is called once per request the client builds, not once per attempt: + // a replayable request whose first attempt stalls or has its connection + // reset is resent once with the headers Authorize set, up to the client's + // overall timeout later (see Client.roundTrip). A provider whose + // credentials are single-use or expire within that window should expect + // the resend to draw a 401, which reaches Refresh like any other. + // // The error must not carry the credential: it travels into the client's own // error text, which reaches logs and terminals. Authorize(ctx context.Context, req *http.Request) error diff --git a/internal/httpclient/wire/stall_retry.go b/internal/httpclient/wire/stall_retry.go new file mode 100644 index 000000000..c0f3a90dc --- /dev/null +++ b/internal/httpclient/wire/stall_retry.go @@ -0,0 +1,261 @@ +package wire + +import ( + "context" + "errors" + "fmt" + "io" + "net/http/httptrace" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +// DefaultResponseHeaderTimeout bounds how long the first attempt of a +// replayable request waits for the server's response HEADERS once the +// request has been written, separately from DefaultTimeout's bound on the +// whole exchange. +// +// It exists for the stuck connection the maintainer-city shadow run found: a +// single HTTP/1.1 connection that went silent while other connections to the +// same serve kept answering, so each request on it sat out the full 60s +// overall timeout. The timer cannot tell that connection from a serve that is +// still working: serve writes a read's headers only once the handler has its +// answer, after waiting up to 10s for a database slot (a wait that ends in a +// 503, which is headers). 15s sits above that wait, but a large read can take +// longer and still be healthy. That is why the retry gets no header timer of +// its own and runs on what is left of the overall timeout (see +// Client.roundTrip): such a read still finishes, later and at the cost of one +// abandoned request, unless the retry's headers and body together take longer +// than what is left, about DefaultTimeout minus this. +// +// Only the headers are bounded: once they arrive the timer is stopped, and a +// slow body that keeps streaming is governed by DefaultTimeout (and the +// caller's context) alone. +// +// The timer applies to replayable requests (see Request.Replayable). A write is +// bounded by the client's overall timeout instead, because bd serve answers a +// write only after it commits; see roundTrip. +const DefaultResponseHeaderTimeout = 15 * time.Second + +// ErrResponseHeaderTimeout is the sentinel a *ResponseHeaderTimeoutError +// unwraps to. +var ErrResponseHeaderTimeout = errors.New("bd serve sent no response headers in time") + +// ResponseHeaderTimeoutError is a request that was written to the server and +// got no response headers in time: its first attempt hit the header timer, or +// the overall timeout ran out while it waited. A request that never left the +// machine (a dial or TLS handshake that timed out) is not one; it stays a +// plain *ConnectError, because the server cannot have seen it. It is returned +// inside a *ConnectError (the server said nothing), so callers that classify +// an unreachable server keep working; errors.As reaches this type for the +// detail. +type ResponseHeaderTimeoutError struct { + Op string + Method string + // Host is the server's host[:port], named so the operator knows which + // server to look at. + Host string + // After is how long the request waited: the header timeout when its timer + // fired, else the measured wait (from when the first attempt was sent, for + // a retried read). + After time.Duration + // Attempts is how many requests went out: 1, or 2 for a read whose retry + // on another connection got no headers either. + Attempts int + // Retried reports whether a retry on another connection was made. + Retried bool + // Reason is why the first attempt was abandoned, when Retried: + // StallHeaderTimeout or StallConnectionReset. + Reason string + // Replayable reports whether the request was idempotent. One that is not + // was never retried and may or may not have been applied. + Replayable bool + // Err is the transport's own error, when the overall timeout (not the + // header timer) is what fired. + Err error +} + +func (e *ResponseHeaderTimeoutError) Error() string { + var b strings.Builder + fmt.Fprintf(&b, "%s: no response headers from %s within %s", e.Op, e.Host, e.After.Round(10*time.Millisecond)) + if e.Retried { + first := "got no headers within the header timeout" + if e.Reason == StallConnectionReset { + first = "had its connection reset" + } + fmt.Fprintf(&b, " on %d attempts (the first %s; the retry used another connection)", e.Attempts, first) + } + if e.Err != nil { + fmt.Fprintf(&b, " (%v)", e.Err) + } + if !e.Replayable { + fmt.Fprintf(&b, "; the %s was not retried because it is not idempotent (it may or may not have been applied)", e.Method) + } + fmt.Fprintf(&b, "; check the server at %s (is it at its connection limit?)", e.Host) + return b.String() +} + +func (e *ResponseHeaderTimeoutError) Unwrap() []error { + if e.Err != nil { + return []error{ErrResponseHeaderTimeout, e.Err} + } + return []error{ErrResponseHeaderTimeout} +} + +// Timeout reports true, so net.Error-style timeout checks classify it. +func (e *ResponseHeaderTimeoutError) Timeout() bool { return true } + +// Temporary is part of the net.Error interface. It is false for a request +// that is not replayable, so a caller that retries whatever reports itself +// temporary never resends a write that may already have been applied. +func (e *ResponseHeaderTimeoutError) Temporary() bool { return e.Replayable } + +// isTimeout reports whether err is a timeout (net.Error, or the client's own +// Timeout surfacing through *url.Error). +func isTimeout(err error) bool { + var te interface{ Timeout() bool } + return errors.As(err, &te) && te.Timeout() +} + +// Reasons a StallRetry names. +const ( + StallHeaderTimeout = "response_header_timeout" + StallConnectionReset = "connection_reset" +) + +// StallRetry describes one retry of an idempotent request on another +// connection after the first attempt stalled (no headers within the header +// timeout) or had its connection reset. It is reported, never swallowed: a retry that +// hides a sick server is a stall nobody sees. +type StallRetry struct { + Op string + Method string + Host string + Reason string + // Retry is the 1-based retry number for this request (always 1 today: + // one retry per request). + Retry int + // Total is the process-wide count of stall retries, this one included. + Total uint64 + // Err is the first attempt's failure. + Err error +} + +func (e StallRetry) String() string { + return fmt.Sprintf("bd: http %s %s to %s stalled (%s); retry %d on another connection (%d stall retries in this process): %v", + e.Op, e.Method, e.Host, e.Reason, e.Retry, e.Total, e.Err) +} + +var stallRetries atomic.Uint64 + +// StallRetryCount is the number of stall retries this process has made. +func StallRetryCount() uint64 { return stallRetries.Load() } + +// stallReason classifies a failed attempt as retryable on another +// connection, returning "" when it is not. +func stallReason(err error) string { + var ce *ConnectError + if !errors.As(err, &ce) { + // Only a failure to get an answer is a stall; a credential source's + // own error that happens to wrap io.EOF is not. + return "" + } + switch { + case errors.Is(err, ErrResponseHeaderTimeout): + return StallHeaderTimeout + case errors.Is(err, syscall.ECONNRESET), errors.Is(err, io.ErrUnexpectedEOF), errors.Is(err, io.EOF): + return StallConnectionReset + } + return "" +} + +// headerTimer bounds one attempt's wait for response headers. It starts once +// the request has been written (httptrace's WroteRequest, where +// http.Transport.ResponseHeaderTimeout starts its own clock), so a slow dial, +// TLS handshake or credential source is never counted as a server that went +// silent. When it fires it cancels the attempt's context, which makes +// net/http close that connection rather than return it to the idle pool, so +// a retry can never ride it. It also records when the request was written, +// which is what tells a write that timed out awaiting its headers from one +// that never left the machine. +type headerTimer struct { + d time.Duration + cancel context.CancelCauseFunc + + mu sync.Mutex + state int // headerWaiting, headerSettled or headerTimedOut + wroteAt time.Time // zero until the request has been written + timer *time.Timer +} + +const ( + headerWaiting = iota + headerSettled // Do returned before the timer fired + headerTimedOut // the timer fired first +) + +// startHeaderTimer returns the attempt's context, which carries the trace hook +// that arms a timer of d (none when d <= 0) once the request is written. +func startHeaderTimer(ctx context.Context, d time.Duration) (context.Context, *headerTimer) { + attemptCtx, cancel := context.WithCancelCause(ctx) + h := &headerTimer{d: d, cancel: cancel} + return httptrace.WithClientTrace(attemptCtx, &httptrace.ClientTrace{WroteRequest: h.wroteRequest}), h +} + +// wroteRequest runs on the transport's writing goroutine. A failed write +// records nothing (the attempt fails on its own), and only the first +// successful one counts: the transport may resend a request it got no answer +// to on a connection it found dead, and the wait is measured from the first. +func (h *headerTimer) wroteRequest(info httptrace.WroteRequestInfo) { + if info.Err != nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + if !h.wroteAt.IsZero() { + return + } + h.wroteAt = time.Now() + if h.d > 0 && h.state == headerWaiting { + h.timer = time.AfterFunc(h.d, h.fire) + } +} + +func (h *headerTimer) fire() { + h.mu.Lock() + fired := h.state == headerWaiting + if fired { + h.state = headerTimedOut + } + h.mu.Unlock() + if fired { + h.cancel(ErrResponseHeaderTimeout) + } +} + +// stop must be called as soon as Do returns. It disarms the timer and reports +// whether it had already fired (the headers lost the race). +func (h *headerTimer) stop() (timedOut bool) { + h.mu.Lock() + defer h.mu.Unlock() + if h.timer != nil { + h.timer.Stop() + } + if h.state == headerWaiting { + h.state = headerSettled + } + return h.state == headerTimedOut +} + +// written reports when the request was written, if it was. +func (h *headerTimer) written() (time.Time, bool) { + h.mu.Lock() + defer h.mu.Unlock() + return h.wroteAt, !h.wroteAt.IsZero() +} + +// done releases the attempt's context. Call it once the body has been read. +func (h *headerTimer) done() { h.cancel(nil) } diff --git a/internal/httpclient/wire/stall_retry_test.go b/internal/httpclient/wire/stall_retry_test.go new file mode 100644 index 000000000..26f8d7079 --- /dev/null +++ b/internal/httpclient/wire/stall_retry_test.go @@ -0,0 +1,775 @@ +package wire + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "slices" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/net/netutil" +) + +// stallServer answers every request normally except those that arrive on the +// FIRST connection it ever sees: those get no response headers at all, the +// shape of the shadow run's stuck HTTP/1.1 connection (a silent connection +// while other connections to the same serve kept answering). The hung handler +// returns when the client gives up on the connection. +type stallServer struct { + srv *httptest.Server + + mu sync.Mutex + first string // RemoteAddr of the first connection + remotes []string // RemoteAddr of every request, in order + methods []string + hang func(w http.ResponseWriter, r *http.Request) +} + +func newStallServer(t *testing.T, answer http.HandlerFunc) *stallServer { + t.Helper() + s := &stallServer{} + s.hang = func(_ http.ResponseWriter, r *http.Request) { + // Drain the body first: net/http only notices the client closing the + // connection (and cancels r.Context) once the body has been read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + } + s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + if s.first == "" { + s.first = r.RemoteAddr + } + stuck := r.RemoteAddr == s.first + s.remotes = append(s.remotes, r.RemoteAddr) + s.methods = append(s.methods, r.Method) + hang := s.hang + s.mu.Unlock() + if stuck { + hang(w, r) + return + } + answer(w, r) + })) + t.Cleanup(s.srv.Close) + return s +} + +func (s *stallServer) seen() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.remotes...) +} + +func (s *stallServer) client(t *testing.T, opts Options) *Client { + t.Helper() + base, err := url.Parse(s.srv.URL) + if err != nil { + t.Fatal(err) + } + if opts.HTTPClient == nil { + // The production shape: a pooled transport with the package's own 60s + // overall ceiling, which the header timeout must undercut. + opts.HTTPClient = &http.Client{Transport: s.srv.Client().Transport, Timeout: DefaultTimeout} + } + c, err := New(base, nil, opts) + if err != nil { + t.Fatalf("New: %v", err) + } + return c +} + +const testHeaderTimeout = 300 * time.Millisecond + +func TestAStalledIdempotentReadRetriesOnceOnAnotherConnection(t *testing.T) { + s := newStallServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + var events []StallRetry + c := s.client(t, Options{ + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(e StallRetry) { events = append(events, e) }, + }) + before := StallRetryCount() + + start := time.Now() + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v", err) + } + elapsed := time.Since(start) + + if out["ok"] != true { + t.Fatalf("body = %v, want the second connection's answer", out) + } + // One header timeout, then one prompt retry: well under two timeouts, and + // nowhere near the 60s overall ceiling the shadow run hit. + if elapsed < testHeaderTimeout || elapsed > 2*testHeaderTimeout+time.Second { + t.Errorf("elapsed %v, want about one header timeout (%v)", elapsed, testHeaderTimeout) + } + seen := s.seen() + if len(seen) != 2 { + t.Fatalf("server saw %d requests, want 2 (the stalled one and one retry): %v", len(seen), seen) + } + if seen[0] == seen[1] { + t.Errorf("the retry rode the stuck connection %s; it must use another one", seen[0]) + } + if len(events) != 1 { + t.Fatalf("OnStallRetry fired %d times, want 1: %+v", len(events), events) + } + e := events[0] + if e.Op != OpGetContext || e.Method != http.MethodGet || e.Retry != 1 || e.Reason != StallHeaderTimeout { + t.Errorf("event = %+v, want op %s, GET, retry 1, reason %q", e, OpGetContext, StallHeaderTimeout) + } + if !strings.Contains(e.String(), "retry 1") || !strings.Contains(e.String(), e.Host) { + t.Errorf("event line %q should carry the retry count and the host", e.String()) + } + if got := StallRetryCount() - before; got != 1 { + t.Errorf("StallRetryCount advanced by %d, want 1", got) + } +} + +func TestAnExplicitlyIdempotentPostReadRetriesToo(t *testing.T) { + s := newStallServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + c := s.client(t, Options{ResponseHeaderTimeout: testHeaderTimeout}) + err := c.Do(ctx(t), Request{Op: OpBatchGetIssues, Method: http.MethodPost, Path: PathIssuesBatchGet, + Body: map[string]any{"ids": []string{"x"}}, Idempotent: true}, nil) + if err != nil { + t.Fatalf("Do: %v", err) + } + if n := len(s.seen()); n != 2 { + t.Fatalf("server saw %d requests, want 2", n) + } +} + +func TestAStalledWriteIsNeverRetriedAndSurfacesATypedTimeout(t *testing.T) { + s := newStallServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + fired := 0 + // A write is bounded by the client's overall timeout, not the header + // timer; shrink the overall one so the stall surfaces quickly. + c := s.client(t, Options{ + HTTPClient: &http.Client{Transport: s.srv.Client().Transport, Timeout: 2 * testHeaderTimeout}, + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(StallRetry) { fired++ }, + }) + + start := time.Now() + err := c.Do(ctx(t), Request{Op: OpCreateIssue, Method: http.MethodPost, Path: PathIssues, + Body: map[string]any{"title": "t"}}, nil) + elapsed := time.Since(start) + if err == nil { + t.Fatal("Do succeeded; want the header timeout") + } + var hte *ResponseHeaderTimeoutError + if !errors.As(err, &hte) { + t.Fatalf("err = %T %v, want *ResponseHeaderTimeoutError", err, err) + } + if !errors.Is(err, ErrResponseHeaderTimeout) { + t.Errorf("err does not unwrap to ErrResponseHeaderTimeout: %v", err) + } + var ce *ConnectError + if !errors.As(err, &ce) { + t.Errorf("err should still be a *ConnectError for callers that classify unreachable servers: %v", err) + } + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Errorf("err should report Timeout() = true: %v", err) + } + if hte.Retried || hte.Attempts != 1 || hte.Method != http.MethodPost { + t.Errorf("hte = %+v, want one unretried POST attempt", hte) + } + if hte.Temporary() { + t.Error("a write's timeout reports Temporary() = true, so a generic net.Error retry would resend it") + } + // The measured wait since the request was written, not a configured + // number, and the transport's own error rather than nothing. + if hte.After < testHeaderTimeout || hte.After > elapsed || hte.Err == nil { + t.Errorf("hte.After = %v, hte.Err = %v; want the measured wait (at most %v) and the transport's error", hte.After, hte.Err, elapsed) + } + host := s.srv.Listener.Addr().String() + msg := err.Error() + for _, want := range []string{host, "not retried", "may or may not have been applied", "Client.Timeout exceeded", "check the server"} { + if !strings.Contains(msg, want) { + t.Errorf("error %q should contain %q", msg, want) + } + } + if n := len(s.seen()); n != 1 { + t.Fatalf("server saw %d requests, want exactly 1 (a write is never replayed)", n) + } + if fired != 0 { + t.Errorf("OnStallRetry fired %d times for a write", fired) + } + if elapsed < 2*testHeaderTimeout || elapsed > 2*testHeaderTimeout+time.Second { + t.Errorf("elapsed %v; the write should fail at the overall timeout (%v)", elapsed, 2*testHeaderTimeout) + } +} + +// TestASlowWriteIsNotCutOffByTheHeaderTimeout: bd serve answers a write only +// after it commits, so a write whose headers take longer than the header +// timeout is a slow but healthy write, and must complete. +func TestASlowWriteIsNotCutOffByTheHeaderTimeout(t *testing.T) { + c, rec := newTestClient(t, Options{ResponseHeaderTimeout: testHeaderTimeout}, nil, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + time.Sleep(3 * testHeaderTimeout) + _, _ = w.Write([]byte(`{}`)) + }) + err := c.Do(ctx(t), Request{Op: OpCreateIssue, Method: http.MethodPost, Path: PathIssues, + Body: map[string]any{"title": "t"}}, nil) + if err != nil { + t.Fatalf("Do: %v", err) + } + if n := rec.count(); n != 1 { + t.Errorf("server saw %d requests, want 1", n) + } +} + +// TestAReadThatStallsTwiceGivesUpAfterOneRetry: the retry has no header timer +// of its own, so a read whose retry gets no headers either fails once the +// overall timeout is spent, counted from the first attempt, and not later. +func TestAReadThatStallsTwiceGivesUpAfterOneRetry(t *testing.T) { + var mu sync.Mutex + n := 0 + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + mu.Lock() + n++ + mu.Unlock() + <-r.Context().Done() + })) + t.Cleanup(srv.Close) + base, _ := url.Parse(srv.URL) + budget := 4 * testHeaderTimeout + c, err := New(base, nil, Options{ + HTTPClient: &http.Client{Transport: srv.Client().Transport, Timeout: budget}, + ResponseHeaderTimeout: testHeaderTimeout, + }) + if err != nil { + t.Fatal(err) + } + start := time.Now() + err = c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, nil) + elapsed := time.Since(start) + var hte *ResponseHeaderTimeoutError + if !errors.As(err, &hte) { + t.Fatalf("err = %v, want *ResponseHeaderTimeoutError", err) + } + if !hte.Retried || hte.Attempts != 2 || hte.Reason != StallHeaderTimeout || !hte.Temporary() { + t.Errorf("hte = %+v, want 2 attempts with the retry and its reason recorded", hte) + } + if elapsed < budget || elapsed > budget+time.Second { + t.Errorf("elapsed %v, want the overall timeout (%v) and no more", elapsed, budget) + } + if hte.After < budget || hte.After > elapsed { + t.Errorf("hte.After = %v, want the whole wait (between %v and %v)", hte.After, budget, elapsed) + } + if msg := err.Error(); !strings.Contains(msg, "on 2 attempts (the first got no headers within the header timeout") { + t.Errorf("error %q should say both attempts went unanswered", msg) + } + mu.Lock() + defer mu.Unlock() + if n != 2 { + t.Errorf("server saw %d requests, want 2", n) + } +} + +func TestAResetConnectionOnAReadRetriesOnAnotherConnection(t *testing.T) { + s := newStallServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + // The first connection is reset (RST, not FIN) instead of hanging. + s.hang = resetConnection + var events []StallRetry + c := s.client(t, Options{ResponseHeaderTimeout: testHeaderTimeout, OnStallRetry: func(e StallRetry) { events = append(events, e) }}) + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v", err) + } + if out["ok"] != true { + t.Fatalf("body = %v", out) + } + if len(events) != 1 || events[0].Reason != StallConnectionReset { + t.Fatalf("events = %+v, want one %q retry", events, StallConnectionReset) + } +} + +// resetConnection is a hang that resets the connection (RST, not FIN). +func resetConnection(w http.ResponseWriter, _ *http.Request) { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + if tc, ok := conn.(*net.TCPConn); ok { + _ = tc.SetLinger(0) + } + _ = conn.Close() +} + +func TestARetryAfterAResetNamesTheReset(t *testing.T) { + s := newStallServer(t, func(_ http.ResponseWriter, r *http.Request) { + <-r.Context().Done() // the retry's connection never answers either + }) + s.hang = resetConnection + c := s.client(t, Options{ + HTTPClient: &http.Client{Transport: s.srv.Client().Transport, Timeout: 2 * testHeaderTimeout}, + ResponseHeaderTimeout: testHeaderTimeout, + }) + err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, nil) + var hte *ResponseHeaderTimeoutError + if !errors.As(err, &hte) { + t.Fatalf("err = %v, want *ResponseHeaderTimeoutError", err) + } + if !hte.Retried || hte.Reason != StallConnectionReset { + t.Errorf("hte = %+v, want the retry recorded with reason %q", hte, StallConnectionReset) + } + if msg := err.Error(); !strings.Contains(msg, "the first had its connection reset") { + t.Errorf("error %q should name the first attempt's reset", msg) + } +} + +// TestASlowReadFinishesOnItsRetry: the timer cannot tell a silent connection +// from a serve still working on a slow read. A read slower than the timer on +// every connection is retried once and, with no timer on the retry, still +// finishes, as it did before there was a timer. +func TestASlowReadFinishesOnItsRetry(t *testing.T) { + var mu sync.Mutex + n := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + n++ + mu.Unlock() + select { + case <-time.After(2 * testHeaderTimeout): + case <-r.Context().Done(): + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + base, _ := url.Parse(srv.URL) + var events []StallRetry + c, err := New(base, nil, Options{ + HTTPClient: &http.Client{Transport: srv.Client().Transport, Timeout: DefaultTimeout}, + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(e StallRetry) { events = append(events, e) }, + }) + if err != nil { + t.Fatal(err) + } + start := time.Now() + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v; a read slower than the header timeout must still finish", err) + } + if out["ok"] != true { + t.Fatalf("body = %v", out) + } + if elapsed := time.Since(start); elapsed < 3*testHeaderTimeout { + t.Errorf("elapsed %v, want one header timeout plus the slow answer (%v)", elapsed, 3*testHeaderTimeout) + } + if len(events) != 1 || events[0].Reason != StallHeaderTimeout { + t.Errorf("events = %+v, want one %q retry", events, StallHeaderTimeout) + } + mu.Lock() + defer mu.Unlock() + if n != 2 { + t.Errorf("server saw %d requests, want 2 (the abandoned attempt and the retry)", n) + } +} + +// TestAReadQueuedBehindTheConnectionLimitFinishesOnceASlotFrees is serve at its +// connection limit (a netutil.LimitListener, as serve's own): the kernel +// completes the handshake and takes the request, but serve accepts the +// connection only once a slot frees. The retry waits for that within the +// overall timeout instead of failing a read serve was about to answer. +func TestAReadQueuedBehindTheConnectionLimitFinishesOnceASlotFrees(t *testing.T) { + occupied := make(chan struct{}) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/occupy" { + close(occupied) + time.Sleep(3 * testHeaderTimeout) + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + srv.Listener = netutil.LimitListener(srv.Listener, 1) + // Every answer closes its connection, so the slot frees once it is sent. + srv.Config.SetKeepAlivesEnabled(false) + srv.Start() + t.Cleanup(srv.Close) + + occupier := make(chan error, 1) + go func() { + resp, err := srv.Client().Get(srv.URL + "/occupy") + if err == nil { + _ = resp.Body.Close() + } + occupier <- err + }() + <-occupied + + base, _ := url.Parse(srv.URL) + var events []StallRetry + c, err := New(base, nil, Options{ + HTTPClient: &http.Client{Transport: srv.Client().Transport, Timeout: DefaultTimeout}, + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(e StallRetry) { events = append(events, e) }, + }) + if err != nil { + t.Fatal(err) + } + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v; a read queued behind the connection limit must finish once a slot frees", err) + } + if out["ok"] != true { + t.Fatalf("body = %v", out) + } + if len(events) != 1 || events[0].Reason != StallHeaderTimeout { + t.Errorf("events = %+v, want one %q retry", events, StallHeaderTimeout) + } + if err := <-occupier; err != nil { + t.Fatalf("the request holding the slot failed: %v", err) + } +} + +// slowCredentials is a credential source slower than the header timer, such as +// a cold BEADS_HTTP_TOKEN_COMMAND. +type slowCredentials struct{ d time.Duration } + +func (s slowCredentials) Authorize(ctx context.Context, _ *http.Request) error { + select { + case <-time.After(s.d): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (slowCredentials) Refresh(context.Context) (bool, error) { return false, nil } + +// TestASlowCredentialSourceIsNotAStall: the timer starts once the request has +// been written, so the time a credential source takes is not counted as a +// server that went silent. +func TestASlowCredentialSourceIsNotAStall(t *testing.T) { + fired := 0 + c, rec := newTestClient(t, Options{ResponseHeaderTimeout: testHeaderTimeout, OnStallRetry: func(StallRetry) { fired++ }}, + slowCredentials{d: 2 * testHeaderTimeout}, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, nil); err != nil { + t.Fatalf("Do: %v", err) + } + if fired != 0 { + t.Errorf("OnStallRetry fired %d times for a slow credential source", fired) + } + if n := rec.count(); n != 1 { + t.Errorf("server saw %d requests, want 1", n) + } +} + +// TestASlowCredentialSourceDoesNotShortenTheRetry: both attempts share the +// client's Timeout counted, as that Timeout is, from when the first attempt is +// sent, so a credential source that takes the whole Timeout to resolve (a cold +// token command) leaves the retry of a slow but healthy read all of what the +// first attempt did not use. The retry replays the stamped request rather than +// consulting the source again, so a source this slow on every call does not +// cost the retry its budget either. +func TestASlowCredentialSourceDoesNotShortenTheRetry(t *testing.T) { + var mu sync.Mutex + var arrived []time.Time + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + arrived = append(arrived, time.Now()) + mu.Unlock() + select { + case <-time.After(2 * testHeaderTimeout): + case <-r.Context().Done(): + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + base, _ := url.Parse(srv.URL) + budget := 5 * testHeaderTimeout + var events []StallRetry + c, err := New(base, slowCredentials{d: budget}, Options{ + HTTPClient: &http.Client{Transport: srv.Client().Transport, Timeout: budget}, + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(e StallRetry) { events = append(events, e) }, + }) + if err != nil { + t.Fatal(err) + } + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v; the credential source's time is not the server's, and must not cut the retry short", err) + } + done := time.Now() + if out["ok"] != true { + t.Fatalf("body = %v", out) + } + if len(events) != 1 || events[0].Reason != StallHeaderTimeout { + t.Errorf("events = %+v, want one %q retry", events, StallHeaderTimeout) + } + mu.Lock() + defer mu.Unlock() + if len(arrived) != 2 { + t.Fatalf("server saw %d requests, want 2 (the abandoned attempt and the retry)", len(arrived)) + } + if took := done.Sub(arrived[0]); took > budget { + t.Errorf("the read finished %v after its first attempt arrived, want within one Timeout (%v)", took, budget) + } +} + +// TestAWriteWhoseDialTimesOutIsAPlainConnectError: a timeout before the request +// was written (here the dial) cannot have reached the server, so it is not +// reported as a header timeout that may or may not have been applied. +func TestAWriteWhoseDialTimesOutIsAPlainConnectError(t *testing.T) { + base, _ := url.Parse("http://127.0.0.1:1") + transport := &http.Transport{DialContext: func(context.Context, string, string) (net.Conn, error) { + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded} + }} + c, err := New(base, nil, Options{ + HTTPClient: &http.Client{Transport: transport, Timeout: DefaultTimeout}, + ResponseHeaderTimeout: testHeaderTimeout, + }) + if err != nil { + t.Fatal(err) + } + err = c.Do(ctx(t), Request{Op: OpCreateIssue, Method: http.MethodPost, Path: PathIssues, + Body: map[string]any{"title": "t"}}, nil) + var ce *ConnectError + if !errors.As(err, &ce) { + t.Fatalf("err = %T %v, want *ConnectError", err, err) + } + var hte *ResponseHeaderTimeoutError + if errors.As(err, &hte) { + t.Fatalf("a write whose dial timed out was reported as a header timeout: %v", err) + } + if msg := err.Error(); !strings.Contains(msg, "i/o timeout") || strings.Contains(msg, "may or may not") { + t.Errorf("error %q should be the dial timeout, not a maybe-applied write", msg) + } +} + +// TestAStallLeavesOtherServersIdleConnectionsAlone: a transport may be shared by +// every client in the process (DialWith's baseline transport), so a stall +// retry must not flush its idle pool: another server's client keeps its +// kept-alive connection. +func TestAStallLeavesOtherServersIdleConnectionsAlone(t *testing.T) { + stalled := newStallServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + var mu sync.Mutex + var remotes []string + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + remotes = append(remotes, r.RemoteAddr) + mu.Unlock() + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(other.Close) + + shared := &http.Transport{} + t.Cleanup(shared.CloseIdleConnections) + hc := &http.Client{Transport: shared, Timeout: DefaultTimeout} + stalledClient := stalled.client(t, Options{HTTPClient: hc, ResponseHeaderTimeout: testHeaderTimeout}) + otherBase, _ := url.Parse(other.URL) + otherClient, err := New(otherBase, nil, Options{HTTPClient: hc, ResponseHeaderTimeout: testHeaderTimeout}) + if err != nil { + t.Fatal(err) + } + + read := Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext} + if err := otherClient.Do(ctx(t), read, nil); err != nil { + t.Fatalf("other server: %v", err) + } + if err := stalledClient.Do(ctx(t), read, nil); err != nil { + t.Fatalf("stalled server: %v", err) + } + if n := len(stalled.seen()); n != 2 { + t.Fatalf("stalled server saw %d requests, want the stall and its retry", n) + } + if err := otherClient.Do(ctx(t), read, nil); err != nil { + t.Fatalf("other server again: %v", err) + } + mu.Lock() + defer mu.Unlock() + if len(remotes) != 2 || remotes[0] != remotes[1] { + t.Errorf("other server saw connections %v; its kept-alive connection should have survived the stall retry", remotes) + } +} + +func TestASlowStreamingBodyIsNotCutOffByTheHeaderTimeout(t *testing.T) { + c, rec := newTestClient(t, Options{ResponseHeaderTimeout: testHeaderTimeout}, nil, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"ok":true`) + w.(http.Flusher).Flush() + // Keep streaming for well past the header timeout. + for range 8 { + time.Sleep(testHeaderTimeout / 2) + _, _ = io.WriteString(w, " ") + w.(http.Flusher).Flush() + } + _, _ = io.WriteString(w, `}`) + }) + start := time.Now() + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v", err) + } + if out["ok"] != true { + t.Fatalf("body = %v", out) + } + if elapsed := time.Since(start); elapsed < 3*testHeaderTimeout { + t.Errorf("elapsed %v: the body should have streamed past the header timeout", elapsed) + } + if n := rec.count(); n != 1 { + t.Errorf("server saw %d requests, want 1 (no retry for a slow body)", n) + } +} + +func TestTheDefaultHeaderTimeoutUndercutsTheOverallTimeout(t *testing.T) { + if DefaultResponseHeaderTimeout <= 0 || DefaultResponseHeaderTimeout >= DefaultTimeout { + t.Fatalf("DefaultResponseHeaderTimeout = %v, want it positive and below DefaultTimeout (%v)", + DefaultResponseHeaderTimeout, DefaultTimeout) + } +} + +func TestANegativeHeaderTimeoutDisablesIt(t *testing.T) { + c, _ := newTestClient(t, Options{ResponseHeaderTimeout: -1}, nil, func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(2 * testHeaderTimeout) + _, _ = w.Write([]byte(`{}`)) + }) + if c.headerTimeout != 0 { + t.Fatalf("headerTimeout = %v, want disabled", c.headerTimeout) + } + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, nil); err != nil { + t.Fatalf("Do: %v", err) + } +} + +// TestAReadThatSpentTheWholeTimeoutIsNotRetried: the retry runs on what the +// first attempt left of the overall timeout, so a read whose first attempt used +// all of it (here with the header timer off, as +// BEADS_HTTP_RESPONSE_HEADER_TIMEOUT=off leaves it) fails with its typed +// timeout, and reports no retry it has no time left to make. +func TestAReadThatSpentTheWholeTimeoutIsNotRetried(t *testing.T) { + var mu sync.Mutex + n := 0 + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + mu.Lock() + n++ + mu.Unlock() + <-r.Context().Done() + })) + t.Cleanup(srv.Close) + base, _ := url.Parse(srv.URL) + budget := testHeaderTimeout + fired := 0 + c, err := New(base, nil, Options{ + HTTPClient: &http.Client{Transport: srv.Client().Transport, Timeout: budget}, + ResponseHeaderTimeout: -1, + OnStallRetry: func(StallRetry) { fired++ }, + }) + if err != nil { + t.Fatal(err) + } + start := time.Now() + err = c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, nil) + elapsed := time.Since(start) + var hte *ResponseHeaderTimeoutError + if !errors.As(err, &hte) { + t.Fatalf("err = %T %v, want *ResponseHeaderTimeoutError", err, err) + } + if hte.Retried || hte.Attempts != 1 || hte.Err == nil { + t.Errorf("hte = %+v, want one unretried attempt and the transport's timeout", hte) + } + if fired != 0 { + t.Errorf("OnStallRetry fired %d times with none of the overall timeout left", fired) + } + if elapsed < budget || elapsed > budget+time.Second { + t.Errorf("elapsed %v, want the overall timeout (%v) and no more", elapsed, budget) + } + mu.Lock() + defer mu.Unlock() + if n != 1 { + t.Errorf("server saw %d requests, want 1", n) + } +} + +// countingToken is a staticToken that counts its Authorize calls. +type countingToken struct { + staticToken + authorized int +} + +func (c *countingToken) Authorize(ctx context.Context, req *http.Request) error { + c.authorized++ + return c.staticToken.Authorize(ctx, req) +} + +// TestAStallRetryReplaysTheCredentialAndA401StillRefreshes: a read whose first +// attempt stalls is resent with the headers Authorize set for it, not +// authorized again, so a credential that went stale meanwhile draws a 401 on +// the retry. That 401 reaches Refresh like any other, and the request Refresh +// re-issues is authorized afresh. +func TestAStallRetryReplaysTheCredentialAndA401StillRefreshes(t *testing.T) { + var mu sync.Mutex + var answered []string + s := newStallServer(t, func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + mu.Lock() + answered = append(answered, auth) + mu.Unlock() + if auth != "Bearer rotated" { + unauthenticated(w) + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + creds := &countingToken{staticToken: staticToken{tokens: []string{"stale", "rotated"}, retry: true}} + base, _ := url.Parse(s.srv.URL) + var events []StallRetry + c, err := New(base, creds, Options{ + HTTPClient: &http.Client{Transport: s.srv.Client().Transport, Timeout: DefaultTimeout}, + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(e StallRetry) { events = append(events, e) }, + }) + if err != nil { + t.Fatal(err) + } + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v", err) + } + if out["ok"] != true { + t.Fatalf("body = %v, want the answer to the refreshed credential", out) + } + if len(events) != 1 || events[0].Reason != StallHeaderTimeout { + t.Errorf("events = %+v, want one %q retry", events, StallHeaderTimeout) + } + if creds.authorized != 2 { + t.Errorf("Authorize called %d times, want 2 (once per request built, not once per attempt)", creds.authorized) + } + if creds.refreshN != 1 { + t.Errorf("Refresh called %d times, want 1", creds.refreshN) + } + mu.Lock() + defer mu.Unlock() + if want := []string{"Bearer stale", "Bearer rotated"}; !slices.Equal(answered, want) { + t.Errorf("answered requests carried %q, want %q (the stall retry, then the refreshed request)", answered, want) + } + if n := len(s.seen()); n != 3 { + t.Errorf("server saw %d requests, want 3 (the stall, its retry, and the refreshed request)", n) + } +} diff --git a/internal/httpclient/wire/stream.go b/internal/httpclient/wire/stream.go index bf6fb3cbb..6d22364f9 100644 --- a/internal/httpclient/wire/stream.go +++ b/internal/httpclient/wire/stream.go @@ -35,6 +35,14 @@ import ( // pieces: the shared request stamp, redirectRefused, the bounded error-body // read, mapProblem. Only once the stream is open does the vocabulary become // SSE. +// +// What the connect does NOT share is roundTrip's stall handling: it gets no +// response-header timer and no retry on another connection (see +// Options.ResponseHeaderTimeout). With the timeout removed, a connect that +// lands on a connection gone silent waits until the caller's context ends, as +// every connect did before the timer existed. The timer was scoped to Do's +// reads, where the shadow run's stall cost each read the full 60s; bounding +// this connect is a separate change. // watchIdleTimeout is how long a stream may go silent before this client // declares it dead and fails loudly. From eaed71fbba485418b2e8974fa7d17c0ef037190a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 14 Oct 2026 05:51:53 +0000 Subject: [PATCH 06/76] fix(http): bd promote over http through issueops.Promote and issues.update.persistence (S21b) (#7554) * fix(http): bd promote over http through issueops.Promote and issues.update.persistence (S21b) `bd promote` refused over a connected HTTP workspace: it called the raw PromoteFromEphemeral, which the http backend stubs, and updateIssue's patch did not publish persistence. MC's PR-intake formula (mol-normalized-issue-pr-v1) runs `gc bd promote --reason ...`, so this blocked the HTTP cutover. - One library entry, issueops.Promote(ctx, PromoteRoles{Reader, Lifecycle, Commenter}, PromoteRequest{Actor, IssueID, Reason, Provenance}): read the row, refuse a non-wisp (*NotAWispError, the sentence bd promote always printed), move it with ONE Lifecycle.Update whose patch is Persistence = persistent (the role's atomic plane move: same id, labels, deps, events and comments carried), then write the promotion comment (a comment failure is reported, the promotion stands). A no-op move is a lost race and reports NotAWispError without a second comment. - cmd/bd: the direct route runs it over the store's own role accessors, so a connected http workspace and a local one take the same entry. The history label is "bd: promote " where the store carries caller labels. - bd serve: IssuePatchBody.persistence (persistent|ephemeral|no_history, anything else a 400 naming patch.persistence) on PATCH /v0/beads/issues/{id}, behind the new additive behavior token issues.update.persistence. wire_revision unchanged. Spec, apigen and the wire-shape golden regenerated; the handler only decodes and passes it to the role. - httpclient: Lifecycle.Update sends patch.persistence only to a server advertising the token, and otherwise refuses BEFORE the dial with *wire.CapabilityError naming it (unwraps to *storage.ErrUnsupported with Capability set). The batchApply item still refuses it (W-IssuePatch.Persistence now reads "ON issues:batchApply"). - Conformance: RunLifecycleUpdatePersistentPromotesAWisp on dolt, embedded, uow and the served http leg; the served PersistentPreservesUnversionedClass case is unparked. - Served e2e TestE2E_PromoteOverHTTP: the mc-promote shape end to end (label kept, comment written, one PATCH, re-promote refused), and under the MC server mask a pre-dial refusal naming the token with no write. - Corpus: mc-promote stays pending, relabelled SRV-1 with the reference profile's exact refusal (the 1.3.0 server lacks issues.update.persistence). Co-Authored-By: Claude Opus 5.5 * fix(promote): run the proxied-server route through issueops.Promote (bd-4y73m9) The adoption review found (major) that the proxied-server route still promoted through the old engine, IssueUseCase.PromoteWisp in one transaction with the comment, while the PR's docs said every route runs issueops.Promote. Converge rather than reword: runPromoteProxiedServer resolves the id, then runs issueops.Promote over the provider's own notifying role accessors (proxiedIssueReader, proxiedIssueLifecycle, proxiedCommenter), the entry the direct route runs. Both routes now share promoteFailure (the error mapping) and reportPromotion (the comment warning and the output), so neither contract can drift. What a script sees change on the proxied-server route: - the move is the persistence branch of Lifecycle.Update, so the child counter moves with the row: a promoted parent's next `bd create --parent` mints .3 after .1/.2 instead of failing with "issue already exists"; - on_update fires twice, for the move and for the comment, as on the direct route (it fired once, for the comment); - a promotion is two Dolt commits, "bd: promote " and "bd: comment " (it was one); - a comment that cannot be written leaves the promotion standing behind a warning, as on every other route, instead of rolling it back; - a dual-plane id is refused rather than merged, the update role's template guard applies, and a lost race reads " is not a wisp (already persistent)". The PromoteWisp exemption in TestRecordingIssueUseCaseCoversItsSurface stays, with its reason corrected: no route calls PromoteWisp now, and its DoltStorage twin fires no hook either. The doc comments on the old engine (PromoteFromEphemeral on DoltStore and the db layer, PromoteWisp) no longer say bd promote uses it. CHANGELOG: the promote entry now names what a script can see on the direct and proxied-server routes: the hook firings, the comment row `bd comments` lists, the child-counter carry, the two-commit shape, and a comment failure that warns instead of rolling back. Tests: - TestEmbeddedPromoteMovesAWispParent promotes a wisp parent with .1/.2 children, a dependency and a comment under a counting on_update hook: the hook fires twice, the dependency and the children's parent edge survive, `bd comments` lists the old comment and the promotion reason, and the next child is .3. At the pre-PR base it fails (the hook fired 0 times). - The proxied journey pins the two commits, and new subtests pin the child counter (the child_counters row, next child .3) and the two hook firings. Against the contributor head each fails: one commit, no counter row and a failing create, one firing. Review iteration 2: - TestEmbeddedPromoteMovesAWispParent gets its row in the embedded cmd shard manifest (shard 64 of 100, from the generator's incremental --write). Without it //scripts:go_test_sources_test fails both the NotStale and the bare-write guard, and the generator's --check fails. - notifying.go no longer lists the promotion comment as a divergence. The direct route now writes that comment through st.Commenter(), which HookFiringStore wraps, so an embedded promote fires on_update for it (TestEmbeddedPromoteMovesAWispParent counts both firings). The list counts four, and the empty-commit note becomes the fifth item. - batchapplier.go no longer says persistence is absent from both patch bodies: the update body publishes it behind issues.update.persistence, and only batchApply refuses it (W-IssuePatch.Persistence). - Three comments still tied promotion to the old engine: the conformance fixture's PromoteWisp no longer claims to do what `bd promote` does, the CLI differential's promote scenario names issueops.Promote, and export_source's wisp-relations note names issueops.MoveIssuePersistenceInTx. - The Promote unit test compares the whole update patch with reflect.DeepEqual. The helper it replaces checked 9 of the patch's 22 members, so a Promote that also set Description passed it. Review iteration 3: - A refused persistence move answered 500 `internal` over http and emitted request_error, though the caller could fix the request. The role now returns two refusals as ErrValidation, so bd serve answers 400 invalid_argument and the client maps that back to ErrValidation: an unversioned row asked for ephemeral or no_history (the role wraps types.ErrUnversionedWispMove, a new sentinel NormalizePersistenceMode returns for that case alone, with its message unchanged), and a demotion of a row with retained snapshots. A row that breaks its own persistence invariants still answers 500. - RunLifecycleUpdateWispModesRefuseAnUnversionedRow pins the refusal on every leg (dolt, embedded dolt, the unit of work, served http): both wisp modes match ErrValidation and leave the row durable, unversioned and off the wisp plane. Without the wrap the dolt leg fails, and the served leg fails on "answered 500 internal". - TestMoveIssuePersistenceInTxRefusesDemotionWithRetainedSnapshots is the first test of the snapshot refusal, and fails without its wrap. The unversioned unit test now requires both sentinels. The types tests pin the sentinel, its unchanged message, and that no row-invariant error matches it: checking the unversioned class ahead of the invariant checks fails them. - CHANGELOG: the promote entry counts five script-visible changes, the fifth being the lost-race refusal sentence, and says the two refusals now answer 400 over bd serve and gain a `validation failed: ` prefix locally. - NotAWispError's doc comment records that a no-history wisp is refused too: the gate reads the Ephemeral flag, as bd promote always has, and `bd update --persistent` is the move for that row. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + CHANGELOG.md | 34 +++ .../conformance/lifecycle_update_contract.go | 93 +++++++- backend/conformance/reader_contract.go | 6 +- backend/conformance/role_bundle_cases.go | 2 + backend/http/BUILD.bazel | 1 + backend/http/corpus/testdata/corpus.yaml | 2 +- backend/http/e2e_promote_test.go | 115 ++++++++++ cmd/bd/export_source.go | 2 +- cmd/bd/promote.go | 119 ++++++---- cmd/bd/promote_embedded_test.go | 105 +++++++++ cmd/bd/promote_proxied_integration_test.go | 79 +++++-- cmd/bd/promote_proxied_server.go | 105 ++++----- engdocs/design/http-divergence-ledger.md | 2 +- internal/httpapi/apigen/types.gen.go | 13 +- internal/httpapi/routes.go | 12 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 39 +++- internal/httpapi/update.go | 14 +- internal/httpapi/update_test.go | 37 +++- .../httpapi/wireshape/testdata/golden.json | 6 + internal/httpclient/BUILD.bazel | 1 + internal/httpclient/batchapplier.go | 17 +- internal/httpclient/encode/ledger.go | 4 +- internal/httpclient/lifecycle.go | 38 +++- .../httpclient/served_claim_lifecycle_test.go | 37 ++-- .../httpclient/update_persistence_test.go | 85 +++++++ internal/httpclient/wire/handshake.go | 10 +- internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/write_ledger_test.go | 6 + internal/httpclient/write_roles_test.go | 4 +- internal/storage/dolt/ephemeral_routing.go | 3 +- .../dolt/issue_operations_persistence_test.go | 37 +++- .../dolt/lifecycle_update_contract_test.go | 6 + internal/storage/domain/db/issue.go | 11 +- internal/storage/domain/issue.go | 6 +- .../lifecycle_update_contract_test.go | 6 + internal/storage/issueops/persistence.go | 15 +- .../uow/lifecycle_update_contract_test.go | 6 + internal/storage/uow/notifying.go | 15 +- internal/storage/uow/notifying_parity_test.go | 5 +- internal/types/storage_class_test.go | 21 +- internal/types/types.go | 11 +- issueops/BUILD.bazel | 2 + issueops/promote.go | 118 ++++++++++ issueops/promote_test.go | 207 ++++++++++++++++++ test/conformance/e2e_test.go | 2 +- 47 files changed, 1275 insertions(+), 191 deletions(-) create mode 100644 backend/http/e2e_promote_test.go create mode 100644 internal/httpclient/update_persistence_test.go create mode 100644 issueops/promote.go create mode 100644 issueops/promote_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index ed764b6b4..ce1179ad6 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -151,6 +151,7 @@ 100 63 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead 100 64 TestEmbeddedGateWaiterCheckLifecycle +100 64 TestEmbeddedPromoteMovesAWispParent 100 65 TestEmbeddedIfRevisionPreflightGoneIsPreconditionFailedOnEveryVerb 100 65 TestEmbeddedSupersede diff --git a/CHANGELOG.md b/CHANGELOG.md index 73857d206..241e92c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `bd promote` works over a connected HTTP workspace (S21b). Its one library + entry, `issueops.Promote(ctx, PromoteRoles{Reader, Lifecycle, Commenter}, + PromoteRequest)`, reads the row, refuses one that is not a wisp + (`*issueops.NotAWispError`, the sentence `bd promote` always printed), moves + it with ONE `Lifecycle.Update` whose patch is `Persistence = persistent` + (labels, dependencies, events, comments and the child counter move with it + in the role's transaction), then records the promotion comment. The direct + and proxied-server routes run it over their own roles, so a local promote + now makes the same plane move `bd update --persistent` makes, honoring + `dolt.auto-commit`. That changes five things a script can see on those + routes: the workspace's `on_update` hook fires twice, once for the move and + once for the comment (the direct route never fired it, the proxied-server + route fired it once); the promotion comment is a row `bd comments` lists + (the direct route recorded only a `commented` event); a promoted parent + keeps its child counter, so its next `bd create --parent` mints the number + after its wisp children's instead of failing with + `issue already exists: .1`; a promotion records two Dolt + commits, `bd: promote ` for the move and `bd: comment ` for the + comment (an embedded workspace recorded one, `bd: promote (auto-commit)`, + and the proxied-server route one, `bd: promote `); and a promote that + loses a race to a concurrent one fails with the not-a-wisp sentence instead + of `promoting : wisp not found`. A comment that cannot + be written leaves the promotion standing behind a warning on every route; + the proxied-server route used to roll the promotion back. `bd serve` + publishes the move as `patch.persistence` on `PATCH /v0/beads/issues/{id}` + behind the new `issues.update.persistence` capability (additive; + `wire_revision` is unchanged), which also carries `bd update --persistent`, + `--ephemeral` and `--no-history` over HTTP. Against a server without the + token the client refuses before the dial, naming it; nothing is written. A + move the workspace refuses (an `unversioned`-class row asked for + `--ephemeral` or `--no-history`, or a demotion of a row with retained + snapshots) matches `ErrValidation` on every route: `bd serve` answers it + `400 invalid_argument`, and the local refusal gains a `validation failed: ` + prefix. - `issueops.LeaseReclaimer`, the stale-lease sweep behind `bd reclaim`, is a role on every leg (dolt, embedded dolt, the unit-of-work provider and the HTTP client), and `bd serve` publishes it as `POST /v0/beads/issues:reclaim` diff --git a/backend/conformance/lifecycle_update_contract.go b/backend/conformance/lifecycle_update_contract.go index d58cc0536..ae5e0a420 100644 --- a/backend/conformance/lifecycle_update_contract.go +++ b/backend/conformance/lifecycle_update_contract.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "reflect" + "slices" "sort" "sync" "testing" @@ -1919,10 +1920,10 @@ func RunLifecycleUpdateParentIDReplacesEveryParent(t *testing.T, ctx context.Con // RunLifecycleUpdatePersistentPreservesUnversionedClass pins the half of the // Persistence clause nobody else asserts: "Persistent preserves an existing -// durable unversioned class" (issueops/issueops.go:135-136). The refusal beside -// it — an unversioned row cannot be demoted to a wisp mode — is pinned -// elsewhere; this clause says the legal direction is a no-op that does NOT -// normalize the row into versioned storage. +// durable unversioned class" (issueops/issueops.go:149). The refusal beside +// it — an unversioned row cannot be demoted to a wisp mode — is +// RunLifecycleUpdateWispModesRefuseAnUnversionedRow; this clause says the legal +// direction is a no-op that does NOT normalize the row into versioned storage. // // WHAT A PERSISTENCE RESTATEMENT MUST NOT DO is where the two observations come // in. A persistence MOVE deletes the row from one plane and re-inserts it into @@ -1963,6 +1964,90 @@ func RunLifecycleUpdatePersistentPreservesUnversionedClass(t *testing.T, ctx con assertLifecycleUpdateWispAbsent(t, ctx, fixture, id, "after the persistent restatement") } +// RunLifecycleUpdateWispModesRefuseAnUnversionedRow pins the refusal beside +// that clause: Ephemeral and NoHistory "select wisp retention modes, never +// durable unversioned storage" (issueops/issueops.go:150-151), so an +// unversioned row asked for either is refused. The refusal is the request's, +// a deterministic validation failure, so it matches ErrValidation on every +// leg — over http that is the 400 naming the patch rather than a 500 that +// drops the reason and alerts the operator — and it leaves the row where it +// was: durable, still unversioned, and absent from the wisp plane. +func RunLifecycleUpdateWispModesRefuseAnUnversionedRow(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + + id := fixture.IssuePrefix + "-lup-unversioned-wisp" + seeded := lifecycleUpdateIssue(id) + seeded.StorageClass = types.StorageClassUnversioned + seedLifecycleUpdateIssue(t, ctx, fixture, seeded) + + for _, mode := range []publicops.PersistenceMode{publicops.PersistenceModeEphemeral, publicops.PersistenceModeNoHistory} { + label := "after the refused " + string(mode) + " move" + before := lifecycleUpdateRow(t, ctx, fixture, id) + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{Actor: "writer", IssueID: id, Patch: publicops.IssuePatch{ + Persistence: publicops.Field[publicops.PersistenceMode]{Set: true, Value: mode}, + }}); !errors.Is(err, storage.ErrValidation) { + t.Fatalf("move unversioned %s to %s: error = %v, want ErrValidation", id, mode, err) + } + assertLifecycleUpdateRowUnchanged(t, ctx, fixture, id, label, before) + after := lifecycleUpdateRow(t, ctx, fixture, id) + if after.Ephemeral || after.NoHistory || after.StorageClass != types.StorageClassUnversioned { + t.Errorf("%s %s = {ephemeral %v, no-history %v, class %q}, want the durable unversioned row it was", + id, label, after.Ephemeral, after.NoHistory, after.StorageClass) + } + assertLifecycleUpdateWispAbsent(t, ctx, fixture, id, label) + } +} + +// RunLifecycleUpdatePersistentPromotesAWisp pins the move `bd promote` makes +// (issueops.Promote): Persistence = persistent on an ephemeral wisp is a +// MUTATION that leaves the row on the durable plane under the same id, with +// its labels carried across and nothing left behind on the wisp plane. A +// second identical request is the no-op the "same current mode" clause +// promises, which is how Promote tells a lost race from a promotion. +func RunLifecycleUpdatePersistentPromotesAWisp(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + if fixture.CreateWisp == nil { + t.Skip("fixture has no CreateWisp: this backend has no wisp plane to promote from") + } + + id := fixture.IssuePrefix + "-lup-promote" + seedLifecycleUpdateWisp(t, ctx, fixture, lifecycleUpdateIssue(id, "keep-me")) + if got := lifecycleUpdateRow(t, ctx, fixture, id); !got.Ephemeral { + t.Fatalf("seeded %s is not ephemeral; the case has nothing to promote", id) + } + + promote := publicops.UpdateRequest{Actor: "writer", IssueID: id, Patch: publicops.IssuePatch{ + Persistence: publicops.Field[publicops.PersistenceMode]{Set: true, Value: publicops.PersistenceModePersistent}, + }} + promoted, err := fixture.Lifecycle.Update(ctx, promote) + if err != nil { + t.Fatalf("promote %s: %v", id, err) + } + if !promoted.Changed { + t.Errorf("promoting wisp %s reported Changed = false, want a mutation", id) + } + if promoted.Issue == nil || promoted.Issue.ID != id || promoted.Issue.Ephemeral { + t.Errorf("promote result issue = %+v, want %s on the durable plane", promoted.Issue, id) + } + + after := lifecycleUpdateRow(t, ctx, fixture, id) + if after.Ephemeral || after.NoHistory { + t.Errorf("%s came back {ephemeral %v, no-history %v} after the promotion, want a durable-plane row", id, after.Ephemeral, after.NoHistory) + } + if !slices.Contains(after.Labels, "keep-me") { + t.Errorf("%s labels after the promotion = %v, want the wisp's label carried across", id, after.Labels) + } + assertLifecycleUpdateWispAbsent(t, ctx, fixture, id, "after the promotion") + + again, err := fixture.Lifecycle.Update(ctx, promote) + if err != nil { + t.Fatalf("re-promote %s: %v", id, err) + } + if again.Changed { + t.Errorf("re-promoting durable %s reported Changed = true, want the same-mode no-op", id) + } +} + // RunLifecycleUpdateProvenanceLabelsHistory pins UpdateRequest.Provenance // against the history the backend actually writes // (issueops/issueops.go:261-270): the entry reads as the caller's own string, diff --git a/backend/conformance/reader_contract.go b/backend/conformance/reader_contract.go index d19fea047..2f93f818c 100644 --- a/backend/conformance/reader_contract.go +++ b/backend/conformance/reader_contract.go @@ -113,9 +113,9 @@ type ReaderFixture struct { AddDependency func(context.Context, *types.Dependency, string) error // AddComment puts one comment on a durable issue. AddComment func(ctx context.Context, issueID, author, text string) error - // PromoteWisp moves a seeded wisp into the durable plane, as `bd promote` - // does. It is the only way to seed a DURABLE row of an infra type: creating - // one routes it to the wisps plane on every backend. + // PromoteWisp moves a seeded wisp into the durable plane. It is the only + // way to seed a DURABLE row of an infra type: creating one routes it to the + // wisps plane on every backend. PromoteWisp func(ctx context.Context, id, actor string) error } diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 3453eedaa..caaa5dc7a 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -488,6 +488,8 @@ var roleContractCases = []roleContract{ RunLifecycleUpdateParentIDReplacesTheParentEdge, RunLifecycleUpdateParentIDReplacesEveryParent, RunLifecycleUpdatePersistentPreservesUnversionedClass, + RunLifecycleUpdateWispModesRefuseAnUnversionedRow, + RunLifecycleUpdatePersistentPromotesAWisp, RunLifecycleUpdateProvenanceLabelsHistory, RunLifecycleUpdateRefusesATemplate, RunLifecycleUpdateAllowTemplateEditsATemplate, diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index 045b68860..c2eaf0468 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -35,6 +35,7 @@ go_test( "e2e_export_test.go", "e2e_list_filters_test.go", "e2e_mol_test.go", + "e2e_promote_test.go", "e2e_recompute_blocked_test.go", "e2e_remote_roles_test.go", "e2e_search_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 9c3d2e485..44bf8e392 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -685,7 +685,7 @@ shapes: source: mc mol-normalized-issue-pr-v1 (promote --reason) args: [promote, e2e-wisp-p, --reason, "keep it"] compare: exit - pending: {slice: "S21b", reason: "promote has no served operation (B3 #16): the http Store stubs PromoteFromEphemeral, updateIssue's patch does not publish persistence (W-IssuePatch.Persistence), and no server has a promote route; it needs a new additive capability, then SRV-1"} + pending: {slice: "SRV-1", reason: "the reference server does not advertise issues.update.persistence, so the client refuses at its capability preflight"} - id: mc-mol-burn source: mc review fixtures (mol burn --force) args: [mol, burn, e2e-wisp-m, --force] diff --git a/backend/http/e2e_promote_test.go b/backend/http/e2e_promote_test.go new file mode 100644 index 000000000..0c5062c77 --- /dev/null +++ b/backend/http/e2e_promote_test.go @@ -0,0 +1,115 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestE2E_PromoteOverHTTP runs `bd promote --reason ...` — the shape +// MC's PR-intake formula (mol-normalized-issue-pr-v1) issues, corpus shape +// mc-promote — through the real `bd` binary against an in-process `bd serve` +// (slice S21b). +// +// The CLI runs issueops.Promote, the entry the direct route runs too: one +// updateIssue carrying `patch.persistence: persistent` (the role's plane move, +// in one server transaction) and then the promotion comment. Against a server +// that does not advertise issues.update.persistence — the MC server's +// handshake shape — the client refuses BEFORE the dial, naming the token, and +// nothing is written. +func TestE2E_PromoteOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + + t.Run("advertised", func(t *testing.T) { + proxyAddr, counter := startCountingProxy(t, addr) + f := newHTTPFixture(t, bin, proxyAddr) + wisp := f.create("wisp to keep", "--ephemeral", "-l", "keep-me") + if !showJSONBool(t, f.mustRun(nil, "show", wisp, "--json").stdout, "ephemeral") { + t.Fatalf("created %s is not ephemeral; nothing to promote", wisp) + } + + counter.reset() + r := f.mustRun(nil, "promote", wisp, "--reason", "keep it") + if !strings.Contains(r.stdout, "Promoted "+wisp+" to permanent bead") { + t.Errorf("bd promote output = %q, want the promotion line", r.stdout) + } + if strings.Contains(r.stderr, "Warning") { + t.Errorf("bd promote warned: %s", r.stderr) + } + patches := 0 + for _, p := range counter.snapshot() { + if strings.HasPrefix(p, "PATCH /v0/beads/issues/") { + patches++ + } + } + if patches != 1 { + t.Errorf("bd promote sent %d updateIssue requests, want exactly one: %v", patches, counter.snapshot()) + } + + show := f.mustRun(nil, "show", wisp, "--json").stdout + if showJSONBool(t, show, "ephemeral") { + t.Errorf("%s is still ephemeral after bd promote: %s", wisp, show) + } + if !strings.Contains(show, "keep-me") { + t.Errorf("%s lost its label across the promotion: %s", wisp, show) + } + comments := f.mustRun(nil, "comments", wisp, "--json").stdout + if !strings.Contains(comments, "Promoted from Level 0: keep it") { + t.Errorf("%s carries no promotion comment: %s", wisp, comments) + } + + // A durable row is refused with the sentence bd promote always printed, + // and so is a second promotion of the same row. + again := f.run(nil, "promote", wisp) + if again.code == 0 || !strings.Contains(again.stderr, "is not a wisp (already persistent)") { + t.Errorf("re-promote: exit %d stderr=%s, want the not-a-wisp refusal", again.code, again.stderr) + } + missing := f.run(nil, "promote", "e2e-absent-wisp9") + if missing.code == 0 { + t.Errorf("promote of an absent id exited 0: %s", missing.stdout) + } + }) + + t.Run("MC server mask", func(t *testing.T) { + proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) + f := newHTTPFixture(t, bin, proxyAddr) + wisp := f.create("wisp the old server cannot promote", "--ephemeral") + + counter.reset() + r := f.run(nil, "promote", wisp, "--reason", "keep it") + if r.code == 0 { + t.Fatalf("bd promote against a server without issues.update.persistence exited 0: %s", r.stdout) + } + if !strings.Contains(r.stderr, "issues.update.persistence") { + t.Errorf("refusal does not name the missing capability: %s", r.stderr) + } + for _, p := range counter.snapshot() { + if !strings.HasPrefix(p, "GET ") { + t.Errorf("a refused promote wrote %s; the refusal must come before any write", p) + } + } + if !showJSONBool(t, f.mustRun(nil, "show", wisp, "--json").stdout, "ephemeral") { + t.Errorf("%s left the wisp plane though the promote was refused", wisp) + } + }) +} + +// showJSONBool reads a boolean member of `bd show --json`'s first row; an +// absent member (omitempty) reads false. +func showJSONBool(t *testing.T, out, field string) bool { + t.Helper() + var generic any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &generic); err != nil { + t.Fatalf("parse bd show --json: %v\n%s", err, out) + } + if rows, ok := generic.([]any); ok && len(rows) > 0 { + generic = rows[0] + } + row, _ := generic.(map[string]any) + b, _ := row[field].(bool) + return b +} diff --git a/cmd/bd/export_source.go b/cmd/bd/export_source.go index 8b28b6eaa..d51666c69 100644 --- a/cmd/bd/export_source.go +++ b/cmd/bd/export_source.go @@ -295,7 +295,7 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ // The wisp_labels / wisp_comments rows of a wisp are keyed by an issue_id // that lives in the WISPS table (promotion moves them to the durable - // tables, issueops.PromoteFromEphemeralInTx), so the three wisp readers + // tables, issueops.MoveIssuePersistenceInTx), so the three wisp readers // below only need the ids that are wisps right now. Handing them the // whole workspace was pure waste — on a 19k-issue rig with no wisps it // was three 19k-placeholder statements answering nothing (wy-237yfi). diff --git a/cmd/bd/promote.go b/cmd/bd/promote.go index 44a42eea0..275fe0647 100644 --- a/cmd/bd/promote.go +++ b/cmd/bd/promote.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "os" @@ -11,19 +12,9 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) -// promotionComment is the audit comment both routes record on the promoted -// bead. Shared by the classic and proxied-server paths so the text cannot -// drift. -func promotionComment(reason string) string { - comment := "Promoted from Level 0" - if reason != "" { - comment += ": " + reason - } - return comment -} - // printPromoteResult renders the `bd promote` success output. updated is the // best-effort post-promote re-read used only by --json (it may be nil, in // which case the JSON path prints nothing, matching the classic behavior). @@ -86,39 +77,93 @@ Examples: return HandleErrorRespectJSON("resolving %s: %v", id, err) } - issue, err := store.GetIssue(ctx, fullID) + promoted, err := promoteDirect(ctx, store, fullID, reason) if err != nil { - if errors.Is(err, storage.ErrNotFound) { - return HandleErrorRespectJSON("issue %s not found", fullID) - } - return HandleErrorRespectJSON("getting issue %s: %v", fullID, err) + return promoteFailure(fullID, err) } - if !issue.Ephemeral { - return HandleErrorRespectJSON("%s is not a wisp (already persistent)", fullID) - } - - if err := store.PromoteFromEphemeral(ctx, fullID, currentActor()); err != nil { - return HandleErrorRespectJSON("promoting %s: %v", fullID, err) - } - - // Add promotion comment (issue is now in permanent table, AddComment routes correctly - // via GetIssue fallback) - if err := store.AddComment(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to add promotion comment to %s: %v\n", fullID, err) - } - - commandDidWrite.Store(true) - - var updated *types.Issue - if jsonOutput { - updated, _ = store.GetIssue(ctx, fullID) - } - return printPromoteResult(fullID, updated) + return reportPromotion(fullID, promoted, func() *types.Issue { + updated, _ := store.GetIssue(ctx, fullID) + return updated + }) }, } +// promoteFailure maps a promotion that did not land to the command's error: +// the typed refusal for a row that is not a wisp, the classic sentence for a +// row that is gone, and the id for anything else. Shared by the direct and +// proxied-server routes, which both run issueops.Promote, so the error +// contract cannot drift. +func promoteFailure(fullID string, err error) error { + var notWisp *issueops.NotAWispError + switch { + case errors.As(err, ¬Wisp): + return HandleErrorRespectJSON("%v", err) + case errors.Is(err, storage.ErrNotFound): + return HandleErrorRespectJSON("issue %s not found", fullID) + } + return HandleErrorRespectJSON("promoting %s: %v", fullID, err) +} + +// reportPromotion reports a promotion that landed, for both routes: a comment +// that could not be written is a warning (the move stands, as +// issueops.PromoteResult.CommentErr says), then the success output. reread is +// the best-effort --json re-read, called only under --json. +func reportPromotion(fullID string, promoted issueops.PromoteResult, reread func() *types.Issue) error { + if promoted.CommentErr != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to add promotion comment to %s: %v\n", fullID, promoted.CommentErr) + } + + commandDidWrite.Store(true) + + var updated *types.Issue + if jsonOutput { + updated = reread() + } + return printPromoteResult(fullID, updated) +} + func init() { promoteCmd.Flags().StringP("reason", "r", "", "Reason for promotion") promoteCmd.ValidArgsFunction = issueIDCompletion rootCmd.AddCommand(promoteCmd) } + +// promoteDirect runs `bd promote` on the direct route through +// issueops.Promote, the one library entry every route shares (the +// proxied-server route runs it over the provider's roles, +// runPromoteProxiedServer), over the store's own role accessors (so its +// decorator stack — hooks, telemetry — wraps the move and the comment). On a +// connected http workspace the same entry dials updateIssue with +// `patch.persistence`, and a server without the issues.update.persistence +// capability refuses before the dial, naming it. +// +// The history label is the classic "bd: promote " where the store can +// carry a caller label, and the server's own where it cannot (S6d's rule, +// remote_backend.go). +func promoteDirect(ctx context.Context, st storage.DoltStorage, fullID, reason string) (issueops.PromoteResult, error) { + opsCtx, err := issueOpsContext(ctx) + if err != nil { + return issueops.PromoteResult{}, err + } + reader, err := st.IssueReader() + if err != nil { + return issueops.PromoteResult{}, err + } + lifecycle, err := writeOps(st) + if err != nil { + return issueops.PromoteResult{}, err + } + commenter, err := st.Commenter() + if err != nil { + return issueops.PromoteResult{}, err + } + provenance := "" + if storeCarriesCallerAttribution(st) { + provenance = "bd: promote " + fullID + } + return issueops.Promote(opsCtx, issueops.PromoteRoles{ + Reader: reader, Lifecycle: lifecycle, Commenter: commenter, + }, issueops.PromoteRequest{ + Actor: currentActor(), IssueID: fullID, Reason: reason, Provenance: provenance, + }) +} diff --git a/cmd/bd/promote_embedded_test.go b/cmd/bd/promote_embedded_test.go index b93d3d9cf..4c781779d 100644 --- a/cmd/bd/promote_embedded_test.go +++ b/cmd/bd/promote_embedded_test.go @@ -7,9 +7,11 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" + "time" ) // bdPromote runs "bd promote" with the given args and returns stdout. @@ -40,6 +42,49 @@ func bdPromoteFail(t *testing.T, bd, dir string, args ...string) string { return string(out) } +// plantCountingHook installs the workspace hook name as a script that appends +// its issue id ($1) to a log, one line per firing, and returns the log's path. +func plantCountingHook(t *testing.T, beadsDir, name string) string { + t.Helper() + hookLog := filepath.Join(t.TempDir(), name+".log") + hooksDir := filepath.Join(beadsDir, "hooks") + if err := os.MkdirAll(hooksDir, 0o755); err != nil { + t.Fatalf("mkdir hooks: %v", err) + } + script := "#!/bin/sh\necho \"$1\" >> " + shellQuote(hookLog) + "\n" + if err := os.WriteFile(filepath.Join(hooksDir, name), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant %s: %v", name, err) + } + return hookLog +} + +// waitForHookLines polls a plantCountingHook log until it names id want times. +// The command has returned by then, and a command waits for the hooks it +// fired before it exits, so a count past want is a hook that fired too often. +func waitForHookLines(t *testing.T, hookLog, id string, want int) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for { + data, _ := os.ReadFile(hookLog) // #nosec G304 -- this test's own temp file + got := 0 + for _, line := range strings.Fields(string(data)) { + if line == id { + got++ + } + } + if got > want { + t.Fatalf("hook fired %d times for %s, want %d; log:\n%s", got, id, want, data) + } + if got == want { + return + } + if time.Now().After(deadline) { + t.Fatalf("hook fired %d times for %s, want %d; log:\n%s", got, id, want, data) + } + time.Sleep(50 * time.Millisecond) + } +} + func TestEmbeddedPromoteCLI(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") @@ -153,6 +198,66 @@ func TestEmbeddedPromoteCLI(t *testing.T) { }) } +// TestEmbeddedPromoteMovesAWispParent pins what the direct route's +// persistence move carries for a wisp with children, a dependency and a +// comment. The dependency and the comment reach the durable tables, the +// children keep their parent, and the child counter carries, so the next +// `bd create --parent` mints .3 rather than colliding with the wisp child .1. +// `bd comments` lists the promotion reason, and on_update fires twice: once +// for the move and once for the promotion comment. +func TestEmbeddedPromoteMovesAWispParent(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "pw") + + target := bdCreate(t, bd, dir, "Promote dep target", "--type", "task") + parent := bdCreate(t, bd, dir, "Promoted wisp parent", "--ephemeral") + var children []string + for i := 1; i <= 2; i++ { + child := bdCreate(t, bd, dir, fmt.Sprintf("Wisp child %d", i), "--ephemeral", "--parent", parent.ID) + if want := fmt.Sprintf("%s.%d", parent.ID, i); child.ID != want { + t.Fatalf("wisp child %d is %s, want %s", i, child.ID, want) + } + children = append(children, child.ID) + } + bdDepAdd(t, bd, dir, parent.ID, target.ID) + bdComment(t, bd, dir, parent.ID, "pre-promote note") + + // Planted after the setup, so every line it logs is the promote's. + hookLog := plantCountingHook(t, beadsDir, "on_update") + bdPromote(t, bd, dir, parent.ID, "--reason", "worth keeping") + waitForHookLines(t, hookLog, parent.ID, 2) + + if got := bdShow(t, bd, dir, parent.ID); got.Ephemeral { + t.Errorf("%s still ephemeral after promote", parent.ID) + } + assertDepExists(t, beadsDir, "pw", parent.ID, target.ID) + for _, id := range children { + var shown struct { + Parent *string `json:"parent"` + } + if err := json.Unmarshal(parseShowJSON(t, bdShowJSON(t, bd, dir, id)), &shown); err != nil { + t.Fatalf("decode bd show %s: %v", id, err) + } + if shown.Parent == nil || *shown.Parent != parent.ID { + t.Errorf("%s parent after promote = %v, want %s", id, shown.Parent, parent.ID) + } + } + comments := bdComments(t, bd, dir, parent.ID) + for _, text := range []string{"pre-promote note", "Promoted from Level 0: worth keeping"} { + if !strings.Contains(comments, text) { + t.Errorf("bd comments %s does not list %q:\n%s", parent.ID, text, comments) + } + } + if next := bdCreate(t, bd, dir, "Child after promote", "--parent", parent.ID); next.ID != parent.ID+".3" { + t.Errorf("next child of the promoted parent is %s, want %s", next.ID, parent.ID+".3") + } +} + // TestEmbeddedPromoteCLIConcurrent exercises promote concurrently. func TestEmbeddedPromoteCLIConcurrent(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { diff --git a/cmd/bd/promote_proxied_integration_test.go b/cmd/bd/promote_proxied_integration_test.go index 501ec1c4c..cc65b162e 100644 --- a/cmd/bd/promote_proxied_integration_test.go +++ b/cmd/bd/promote_proxied_integration_test.go @@ -5,6 +5,10 @@ package main import ( "database/sql" "encoding/json" + "fmt" + "runtime" + "slices" + "sort" "strings" "testing" ) @@ -12,9 +16,11 @@ import ( // TestProxiedServerPromote is the promote-family journey on the shared // proxied server. The downstream contract (wyvern wheelhouse portcullis // salvage) is that promote REWRITES THE ROW IN PLACE: same id, wisp_type -// retained, labels/deps/events/comments preserved, still visible to -// `bd list --include-infra --wisp-type `, no longer purge-reclaimable, -// same --json shape and error contract as classic. +// retained, labels/deps/events/comments and the child counter preserved, +// still visible to `bd list --include-infra --wisp-type `, no longer +// purge-reclaimable, same --json shape and error contract as classic. The +// route runs issueops.Promote, the direct route's entry, so it also fires +// on_update for the move and for the promotion comment. func TestProxiedServerPromote(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() @@ -56,16 +62,12 @@ func TestProxiedServerPromote(t *testing.T) { t.Errorf("expected promote success line, got: %s", stdout) } - // Exactly one Dolt commit, carrying the classic commit message. - if n := proxiedDoltCommitCountSince(t, db, head); n != 1 { - t.Errorf("expected exactly 1 Dolt commit for promote, got %d", n) - } - var msg string - if err := db.QueryRow("SELECT message FROM DOLT_LOG('HEAD', '--not', ?)", head).Scan(&msg); err != nil { - t.Fatalf("read promote commit message: %v", err) - } - if msg != "bd: promote "+wisp.ID { - t.Errorf("commit message = %q, want %q", msg, "bd: promote "+wisp.ID) + // Two Dolt commits, one per role transaction: the move under the + // classic message, then the promotion comment under the comment's. + messages := readDoltLogMessagesSince(t, db, head) + sort.Strings(messages) + if want := []string{"bd: comment " + wisp.ID, "bd: promote " + wisp.ID}; !slices.Equal(messages, want) { + t.Errorf("commits since promote = %q, want %q", messages, want) } // The row moved planes in place: same id, flag cleared, type retained. @@ -130,6 +132,57 @@ func TestProxiedServerPromote(t *testing.T) { } }) + // The move carries the child counter, so a promoted wisp parent's next + // child is minted past the children it had as a wisp. The old engine left + // the counter in wisp_child_counters, and the next `bd create --parent` + // minted .1 again and failed on the wisp child holding that id. + t.Run("promoted_parent_keeps_its_child_counter", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "pmc") + parent := bdProxiedCreate(t, bd, p.dir, "Promoted wisp parent", "--ephemeral") + for i := 1; i <= 2; i++ { + child := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("Wisp child %d", i), "--ephemeral", "--parent", parent.ID) + if want := fmt.Sprintf("%s.%d", parent.ID, i); child.ID != want { + t.Fatalf("wisp child %d is %s, want %s", i, child.ID, want) + } + } + + if out, err := bdProxiedRun(t, bd, p.dir, "promote", parent.ID); err != nil { + t.Fatalf("bd promote: %v\n%s", err, out) + } + + db := openProxiedDB(t, p) + if n := countRows(t, db, "SELECT COUNT(*) FROM child_counters WHERE parent_id = ? AND last_child = 2", parent.ID); n != 1 { + t.Errorf("expected the counter (last_child 2) in child_counters, got %d rows", n) + } + if n := countRows(t, db, "SELECT COUNT(*) FROM wisp_child_counters WHERE parent_id = ?", parent.ID); n != 0 { + t.Errorf("expected wisp_child_counters emptied, got %d", n) + } + next := bdProxiedCreate(t, bd, p.dir, "Child after promote", "--parent", parent.ID) + if want := parent.ID + ".3"; next.ID != want { + t.Errorf("next child of the promoted parent is %s, want %s", next.ID, want) + } + }) + + // on_update fires twice, from the notifying provider: once for the move + // and once for the promotion comment. The old engine fired it only for + // the comment. + t.Run("hooks_fire_on_update_for_the_move_and_the_comment", func(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("hook script form is POSIX shell") + } + p := newSharedProxiedProject(t, bd, "pmh") + wisp := bdProxiedCreate(t, bd, p.dir, "Promote hook wisp", "--ephemeral") + // Planted after the setup, so every line it logs is the promote's. + hookLog := plantCountingHook(t, p.beadsDir, "on_update") + + if out, err := bdProxiedRun(t, bd, p.dir, "promote", wisp.ID, "--reason", "hooked"); err != nil { + t.Fatalf("bd promote: %v\n%s", err, out) + } + waitForHookLines(t, hookLog, wisp.ID, 2) + }) + t.Run("json_shape_matches_classic", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "pmj") diff --git a/cmd/bd/promote_proxied_server.go b/cmd/bd/promote_proxied_server.go index 38f6fb547..7deb0471e 100644 --- a/cmd/bd/promote_proxied_server.go +++ b/cmd/bd/promote_proxied_server.go @@ -2,88 +2,77 @@ package main import ( "context" - "errors" - "fmt" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" - "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" ) // runPromoteProxiedServer is the proxied-server route for `bd promote`. It -// mirrors the classic route: resolve the id, refuse a non-wisp with the same -// error text, then promote through the domain PromoteWisp verb — which runs -// the exact issueops implementation the classic route runs, so the row is -// rewritten in place (same id, wisp_type retained, labels/deps/events/ -// comments carried to the permanent tables, inbound edges retargeted). +// runs issueops.Promote, the entry the direct route runs, over the provider's +// own role accessors, so the two routes promote the same way: one +// Lifecycle.Update whose patch is the persistence move (the move +// `bd update --persistent` makes on this route: same id, wisp_type retained, +// labels/deps/events/comments and the child counter carried to the permanent +// tables, inbound edges retargeted), then the promotion comment through the +// Commenter. The hooks fire from the notifying provider (main.go): on_update +// for the move and again for the comment, as on the direct route. // -// Commit semantics: the cross-plane move and the promotion comment land in -// ONE transaction under one Dolt commit ("bd: promote ", the classic -// dolt commit message). The wisp-side deletes touch dolt_ignored tables so -// only the issues-plane rows are versioned by that commit, but the single -// transaction still makes the wisp delete and the issues insert atomic. -// -// One deliberate tightening vs classic: classic adds the promotion comment -// in a separate write after the promote has committed, so a failed comment -// write can only degrade to a stderr warning. Here the comment shares the -// promote's transaction, so a comment failure rolls the whole promote back — -// atomic beats warn-and-continue, and a comment insert on a row this same -// transaction just promoted has no realistic standalone failure mode. +// Each role runs its own transaction, so a promotion is two Dolt commits, the +// move's ("bd: promote ", the classic message) and the comment's +// ("bd: comment "), and a comment that fails leaves the promotion standing +// behind a warning, as on every other route. func runPromoteProxiedServer(ctx context.Context, id, reason string) error { if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } - // Resolve + precondition checks on a read UOW, mirroring the classic - // route's resolve/get/ephemeral-check before its write transaction. The - // promote below re-verifies the wisp is still active inside the write - // transaction (a lost race fails with "wisp not found", as classic). ruw, err := proxiedOpenReadUOW(ctx) if err != nil { return err } fullID, err := utils.ResolvePartialID(ctx, uowMolReader{uw: ruw}, id) - if err != nil { - ruw.Close(ctx) - return HandleErrorRespectJSON("resolving %s: %v", id, err) - } - issue, _, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(ruw), fullID) ruw.Close(ctx) if err != nil { - if errors.Is(err, storage.ErrNotFound) { - return HandleErrorRespectJSON("issue %s not found", fullID) - } - return HandleErrorRespectJSON("getting issue %s: %v", fullID, err) - } - if !issue.Ephemeral { - return HandleErrorRespectJSON("%s is not a wisp (already persistent)", fullID) + return HandleErrorRespectJSON("resolving %s: %v", id, err) } - err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { - if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, currentActor()); err != nil { - return "", err - } - if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { - return "", fmt.Errorf("adding promotion comment: %w", err) - } - return fmt.Sprintf("bd: promote %s", fullID), nil + roles, err := proxiedPromoteRoles() + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + promoted, err := issueops.Promote(ctx, roles, issueops.PromoteRequest{ + Actor: currentActor(), IssueID: fullID, Reason: reason, Provenance: "bd: promote " + fullID, }) if err != nil { - return HandleErrorRespectJSON("promoting %s: %v", fullID, err) + return promoteFailure(fullID, err) } - - commandDidWrite.Store(true) - - var updated *types.Issue - if jsonOutput { - // Best-effort post-promote re-read for --json, matching the classic - // route's ignore-error re-read (labels included, like GetIssueInTx). - if uwr, rerr := proxiedOpenReadUOW(ctx); rerr == nil { - updated, _ = uowMolReader{uw: uwr}.GetIssue(ctx, fullID) - uwr.Close(ctx) + return reportPromotion(fullID, promoted, func() *types.Issue { + // Labels included, like the direct route's store.GetIssue. + uwr, err := proxiedOpenReadUOW(ctx) + if err != nil { + return nil } + defer uwr.Close(ctx) + updated, _ := uowMolReader{uw: uwr}.GetIssue(ctx, fullID) + return updated + }) +} + +// proxiedPromoteRoles asks the provider for the three roles issueops.Promote +// composes, each through its own accessor, so each is the notifying one. +func proxiedPromoteRoles() (issueops.PromoteRoles, error) { + reader, err := proxiedIssueReader() + if err != nil { + return issueops.PromoteRoles{}, err } - return printPromoteResult(fullID, updated) + lifecycle, err := proxiedIssueLifecycle() + if err != nil { + return issueops.PromoteRoles{}, err + } + commenter, err := proxiedCommenter() + if err != nil { + return issueops.PromoteRoles{}, err + } + return issueops.PromoteRoles{Reader: reader, Lifecycle: lifecycle, Commenter: commenter}, nil } diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 01d95c9a3..b3f006887 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -122,7 +122,7 @@ Role-request fields on the served writes that the v0 wire publishes no member fo | W-IssuePatch.AwaitID | refuse | `IssuePatch.AwaitID` | an await-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes await_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-IssuePatch.Owner | refuse | `IssuePatch.Owner` | an owner edit refuses ON updateIssue | IssuePatchBody excludes owner — which the document itself calls an accident of order rather than a decision, since ApplyPatchBody DOES publish it and issues:batchApply's update item carries it (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-IssuePatch.ClosedBySession | refuse | `IssuePatch.ClosedBySession` | a closed-by-session edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes closed_by_session (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-IssuePatch.Persistence | refuse | `IssuePatch.Persistence` | a persistence-mode edit refuses | neither body publishes persistence: moving a row between planes mid-plan is a different act from writing its fields (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Persistence | refuse | `IssuePatch.Persistence` | a persistence-mode edit refuses ON issues:batchApply | ApplyPatchBody excludes persistence: moving a row between planes mid-plan is a different act from writing its fields. IssuePatchBody publishes it behind issues.update.persistence, and updateIssue's encoder carries it (refusing before the dial against a server without that token) (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-IssuePatch.ParentID | retired | — | a parent re-hang through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). It is ONE call rather than a remove-then-add pair, which is the whole reason the member exists: the two-call spelling leaves the issue parentless if the second call fails, and dependencies:add/remove — still the general write side of the graph over this wire — cannot express the atomic replacement. The empty string removes every parent-child edge, so it travels like `assignee`'s empty string rather than being skipped. The graph refusals it earns are the server's and arrive typed: a cycle through the issue's own descendant as the PLAIN dependency_cycle, and an existing edge of another type as dependency_exists | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CreateBatchRequest.Provenance | refuse | `CreateBatchRequest.Provenance` | labeling a batch create's history entry refuses | batchCreateIssues publishes actor and items only, and the server writes its own label; see W-UpdateRequest.Provenance. It is the one Provenance whose ABSENCE is visible in shipped history — the surface this role serves has always named its source file in the entry (`bd: create 3 issue(s) from plan.md`) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CreateBatchRequest.ForceIDPrefix | refuse | `CreateBatchRequest.ForceIDPrefix` `--force` | permitting explicit ids outside the workspace's configured prefix refuses | the wire publishes no such member — and could not honor one, since it publishes no explicit id for the flag to permit (W-BatchCreateItem.Issue). Dropping it would answer a request about WHERE ids may be written with a batch whose ids the server chose | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 7a65483eb..f0f16b608 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -675,7 +675,7 @@ type ApplyMetadataPatch struct { // // `labels` is a full patch rather than a complete replacement, and that one is a real difference: a plan has to be able to REMOVE one label without knowing the rest of the set, because it edits a set it did not compose. A caller patching one row it just read already knows the set. // -// `parent_id` is deliberately absent, and its absence is this operation's one-edge-one-spelling rule: a parent is a `dep_add` item of type `parent-child`, so the order of every edge in the request stays total. The single patch has no ordering to express and publishes it directly. `persistence` is absent from both — moving a row between planes mid-plan is a different act from writing its fields, and nothing has asked for it here. +// `parent_id` is deliberately absent, and its absence is this operation's one-edge-one-spelling rule: a parent is a `dep_add` item of type `parent-child`, so the order of every edge in the request stays total. The single patch has no ordering to express and publishes it directly. `persistence` is absent here — moving a row between planes mid-plan is a different act from writing its fields, and nothing has asked for it here. The single patch publishes it (`IssuePatchBody.persistence`, behind `issues.update.persistence`), where the move is the patch's last step. type ApplyPatchBody struct { AcceptanceCriteria *string `json:"acceptance_criteria,omitempty"` @@ -764,7 +764,7 @@ type ApplyUpdateItem struct { // // `labels` is a full patch rather than a complete replacement, and that one is a real difference: a plan has to be able to REMOVE one label without knowing the rest of the set, because it edits a set it did not compose. A caller patching one row it just read already knows the set. // - // `parent_id` is deliberately absent, and its absence is this operation's one-edge-one-spelling rule: a parent is a `dep_add` item of type `parent-child`, so the order of every edge in the request stays total. The single patch has no ordering to express and publishes it directly. `persistence` is absent from both — moving a row between planes mid-plan is a different act from writing its fields, and nothing has asked for it here. + // `parent_id` is deliberately absent, and its absence is this operation's one-edge-one-spelling rule: a parent is a `dep_add` item of type `parent-child`, so the order of every edge in the request stays total. The single patch has no ordering to express and publishes it directly. `persistence` is absent here — moving a row between planes mid-plan is a different act from writing its fields, and nothing has asked for it here. The single patch publishes it (`IssuePatchBody.persistence`, behind `issues.update.persistence`), where the move is the patch's last step. Patch ApplyPatchBody `json:"patch"` // Target Names ONE issue, either by an id that already exists or by the `key` a create item earlier in the same request gave itself. @@ -1101,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, and `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, and `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1556,7 +1556,12 @@ type IssuePatchBody struct { // // IT IS A GRAPH EDIT, and it earns the graph's refusals: a new parent this workspace holds no row for is a `400`, a pair that already carries an edge of another type is `409 dependency_exists`, and a move under the issue's own descendant is `409 dependency_cycle` — the PLAIN one, carrying no `issue_id`/`blocker_id`/ `blocker_is_ancestor`, because the hierarchy refusal answers only to blocking edges and this member writes a `parent-child` edge. Naming the issue itself is a `400`. One call rather than a remove-then-add pair, which is the whole reason it is here: the two-call spelling leaves the issue parentless if the second call fails. ParentId *string `json:"parent_id,omitempty"` - Priority *int `json:"priority,omitempty"` + + // Persistence One of `persistent`, `ephemeral` or `no_history`; any other value is a `400` naming the member. Selects the issue's complete persistence state, moving the row between the durable and the wisp plane when the state changes plane. `persistent` promotes a wisp: the row keeps its id, and its labels, dependencies, events and comments move with it, all in this update's one transaction (`bd promote`, `bd update --persistent`). `ephemeral` and `no_history` select the wisp retention modes (`bd update --ephemeral` / `--no-history`); demoting a durable row that has retained snapshots is refused. The current mode is a no-op that reports `changed: false`. + // + // A PLANE MOVE IS AN AGGREGATE MIGRATION, not a field write, and it is applied LAST, after every other member of the patch, so the edit and the move commit together or not at all. Advertised by the `issues.update.persistence` capability token; a server that predates the member refuses it as an unknown parameter, and a client that does not see the token refuses the move before the dial rather than sending it. + Persistence *string `json:"persistence,omitempty"` + Priority *int `json:"priority,omitempty"` // RemoveLabels Labels to remove, applied AFTER `labels` and `add_labels`, so REMOVAL WINS over both. // diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 6789f992a..a724540e1 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -221,6 +221,16 @@ const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" // still edits a template (bd-jkp9v3). const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" +// CapIssuesUpdatePersistence is the behavior capability that advertises +// IssuePatchBody.persistence on updateIssue: the role's +// IssuePatch.Persistence, which moves the row between the durable and the wisp +// plane in the update's own transaction. It is what `bd promote` +// (issueops.Promote) and `bd update --persistent/--ephemeral/--no-history` +// send over http. An older server answers the member with `400 +// invalid_argument`/`unknown_parameter`; a client that does not see this token +// refuses the move before the dial, naming the token, rather than sending it. +const CapIssuesUpdatePersistence = "issues.update.persistence" + // CapIssuesCreateDefaultPriority is the behavior capability that advertises // that issues.create, issues.batchCreate and issues.batchApply's create items // store the create default priority (P2) for a request whose `priority` member @@ -1034,7 +1044,7 @@ var behaviorCapabilities = []string{ CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapIssuesCloseAutoCloseMolecule, CapIssuesListSearch, - CapIssuesListNoAssignee, CapIssuesListTitle, CapListIDs, + CapIssuesListNoAssignee, CapIssuesListTitle, CapListIDs, CapIssuesUpdatePersistence, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index d4a6b27f0..43e983b2e 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -639,7 +639,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", "issues.close.autoCloseMolecule", "issues.list.search", - "issues.list.noAssignee", "issues.list.title", "issues.list.ids") + "issues.list.noAssignee", "issues.list.title", "issues.list.ids", "issues.update.persistence") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index de7f04b33..3556c5e99 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2495,11 +2495,6 @@ paths: Nothing has asked for it. - `persistence` — a plane move is an atomic aggregate migration rather - than a field write, and moving a row between planes mid-patch is a - different act from editing it. - - `closed_by_session` and `close_reason` — written under first-close-wins by `{id}:close`, which a patch write would bypass. A `status` that crosses into the done category still answers to close POLICY here; it @@ -8287,6 +8282,11 @@ components: `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and + `issues.update.persistence`, which announces that + `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in + its `patch` (see `IssuePatchBody`), the plane move behind + `bd promote`, rather + than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores @@ -10500,9 +10500,11 @@ components: one-edge-one-spelling rule: a parent is a `dep_add` item of type `parent-child`, so the order of every edge in the request stays total. The single patch has no ordering to express and publishes it directly. - `persistence` is absent from both — moving a row between planes mid-plan + `persistence` is absent here — moving a row between planes mid-plan is a different act from writing its fields, and nothing has asked for it - here. + here. The single patch publishes it (`IssuePatchBody.persistence`, + behind `issues.update.persistence`), where the move is the patch's last + step. properties: title: type: string @@ -12201,6 +12203,29 @@ components: description: 'RFC 3339. Explicit `null` CLEARS the deferral.' metadata: $ref: '#/components/schemas/ApplyMetadataPatch' + persistence: + type: string + maxLength: 255 + description: >- + One of `persistent`, `ephemeral` or `no_history`; any other value + is a `400` naming the member. Selects the issue's complete persistence state, moving the row + between the durable and the wisp plane when the state changes + plane. `persistent` promotes a wisp: the row keeps its id, and its + labels, dependencies, events and comments move with it, all in + this update's one transaction (`bd promote`, `bd update + --persistent`). `ephemeral` and `no_history` select the wisp + retention modes (`bd update --ephemeral` / `--no-history`); + demoting a durable row that has retained snapshots is refused. + The current mode is a no-op that reports `changed: false`. + + + A PLANE MOVE IS AN AGGREGATE MIGRATION, not a field write, and it + is applied LAST, after every other member of the patch, so the + edit and the move commit together or not at all. Advertised by the + `issues.update.persistence` capability token; a server that + predates the member refuses it as an unknown parameter, and a + client that does not see the token refuses the move before the + dial rather than sending it. UpdateIssueResponse: type: object diff --git a/internal/httpapi/update.go b/internal/httpapi/update.go index 64df715ce..2cf299943 100644 --- a/internal/httpapi/update.go +++ b/internal/httpapi/update.go @@ -46,7 +46,7 @@ var ( "title", "description", "design", "acceptance_criteria", "notes", "append_notes", "priority", "issue_type", "status", "assignee", "parent_id", "labels", "add_labels", "remove_labels", "metadata", - "estimated_minutes", "external_ref", "due_at", "defer_until", + "estimated_minutes", "external_ref", "due_at", "defer_until", "persistence", } // nullablePatchMembers is the closed set on which explicit `null` CLEARS // rather than refuses. They are exactly the members the role models as @@ -426,6 +426,18 @@ func (s *Server) issuePatch(w http.ResponseWriter, r *http.Request, id string, m } patch.ParentID = issueops.Field[string]{Set: true, Value: *wire.ParentId} } + if set("persistence") { + // The plane move (CapIssuesUpdatePersistence). The handler checks only + // that the value is one of the role's three modes, so an unknown one is + // a 400 naming the member rather than a translated storage error; the + // move itself, and every refusal it earns, is the role's. + mode := issueops.PersistenceMode(*wire.Persistence) + if !mode.IsValid() { + return refuse("persistence", fmt.Sprintf("`persistence` is %q; send %q, %q or %q", + *wire.Persistence, issueops.PersistenceModePersistent, issueops.PersistenceModeEphemeral, issueops.PersistenceModeNoHistory)) + } + patch.Persistence = issueops.Field[issueops.PersistenceMode]{Set: true, Value: mode} + } if set("metadata") { // applyMetadataPatch, unchanged and not respelled: the replace/merge/ // set/unset algebra is one definition, and a second copy here would be a diff --git a/internal/httpapi/update_test.go b/internal/httpapi/update_test.go index 8dd401a1b..a4fae26d5 100644 --- a/internal/httpapi/update_test.go +++ b/internal/httpapi/update_test.go @@ -305,7 +305,12 @@ func TestUpdateRejectsTheShapesTheDocumentRefuses(t *testing.T) { {"patch is not an object", `{"actor":"alice","patch":["title"]}`, "patch"}, {"empty patch", `{"actor":"alice","patch":{}}`, "patch"}, {"unknown request member", `{"actor":"alice","patch":{"title":"t"},"force":true}`, "force"}, - {"unknown patch member", `{"actor":"alice","patch":{"title":"t","persistence":"ephemeral"}}`, "patch.persistence"}, + {"unknown patch member", `{"actor":"alice","patch":{"title":"t","owner":"bob"}}`, "patch.owner"}, + // persistence is published (issues.update.persistence), but only as + // one of the three modes the role knows. + {"unknown persistence mode", `{"actor":"alice","patch":{"persistence":"durable"}}`, "patch.persistence"}, + {"blank persistence mode", `{"actor":"alice","patch":{"persistence":""}}`, "patch.persistence"}, + {"persistence is not a string", `{"actor":"alice","patch":{"persistence":true}}`, "patch"}, // The refusals the three policy members brought with them, all decided // at the edge because none of them needs to read a row. {"self parent", `{"actor":"alice","patch":{"parent_id":"bd-1"}}`, "patch.parent_id"}, @@ -1383,3 +1388,33 @@ func TestUpdateAnswersARefusedClaimAsTheClaimDoes(t *testing.T) { }) } } + +// TestUpdateForwardsThePersistenceMember is the plane move `bd promote` sends +// (issueops.Promote): patch.persistence reaches the role's +// IssuePatch.Persistence verbatim, for each of the three modes, and nothing +// else is set beside it. +func TestUpdateForwardsThePersistenceMember(t *testing.T) { + for _, mode := range []issueops.PersistenceMode{ + issueops.PersistenceModePersistent, issueops.PersistenceModeEphemeral, issueops.PersistenceModeNoHistory, + } { + t.Run(string(mode), func(t *testing.T) { + lifecycle := &roleLifecycle{updateResult: issueops.UpdateResult{Issue: updatedIssue("bd-1"), Changed: true}} + ts := newUpdateServer(t, lifecycle) + resp := ts.updateIssue(t, updatePath, `{"actor":"alice","patch":{"persistence":"`+string(mode)+`"}}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := lifecycle.updateRequests() + if len(got) != 1 { + t.Fatalf("the role was called %d times, want 1", len(got)) + } + p := got[0].Patch + if !p.Persistence.Set || p.Persistence.Value != mode { + t.Errorf("persistence = %+v, want a set %q", p.Persistence, mode) + } + if p.Title.Set || p.Status.Set || p.Assignee.Set || p.ParentID.Set { + t.Errorf("the handler set a member the body did not carry: %+v", p) + } + }) + } +} diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 14fb03288..b136b13af 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -2917,6 +2917,12 @@ "type": "string", "required": false }, + { + "schema": "IssuePatchBody", + "member": "persistence", + "type": "string", + "required": false + }, { "schema": "IssuePatchBody", "member": "priority", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 6beb6b8ec..68803269c 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -170,6 +170,7 @@ go_test( "store_test.go", "target_test.go", "unlimited_read_test.go", + "update_persistence_test.go", "watch_test.go", "wave2b_roles_test.go", "wave2c_claimnext_test.go", diff --git a/internal/httpclient/batchapplier.go b/internal/httpclient/batchapplier.go index 1fd04b1be..e397f9f99 100644 --- a/internal/httpclient/batchapplier.go +++ b/internal/httpclient/batchapplier.go @@ -482,7 +482,7 @@ func applyRefBody(ref issueops.Ref) (apigen.Ref, error) { // applyPatchExcludedMembers is the refusal half of the apply patch's allowlist, // in the order encodeApplyPatch decides them. // -// THE ALLOWLIST IS NOT updateIssue's, and the two differences run in opposite +// THE ALLOWLIST IS NOT updateIssue's, and the differences run in opposite // directions, which is why this cannot share refuseExcludedPatchMembers: // // owner IS published here and refused there. ApplyPatchBody carries it; @@ -492,17 +492,22 @@ func applyRefBody(ref issueops.Ref) (apigen.Ref, error) { // a parent is a dep_add item of type parent-child, so the order of // every edge in a plan stays total and there is one spelling for // an edge. +// persistence is published THERE, behind issues.update.persistence, and +// absent here, also deliberately: moving a row between planes +// mid-plan is a different act from writing its fields. // -// The other three — spec_id, await_id, closed_by_session — and persistence are -// absent from both bodies. +// The other three — spec_id, await_id, closed_by_session — are absent from +// both bodies. var applyPatchExcludedMembers = []struct { member string ledger string set func(issueops.IssuePatch) bool }{ - // ParentID is this document's OWN exclusion and gets its own row; the other - // four are absent from both patch bodies, so they cite the rows that - // already refuse them rather than minting a second reason for one fact. + // ParentID is this document's OWN exclusion and gets its own row; SpecID, + // AwaitID and ClosedBySession are absent from both patch bodies, so they + // cite the rows that already refuse them rather than minting a second + // reason for one fact. Persistence cites W-IssuePatch.Persistence, the row + // that refuses it on this operation alone. {"ParentID", "W-ApplyPatch.ParentID", func(p issueops.IssuePatch) bool { return p.ParentID.Set }}, {"SpecID", "W-IssuePatch.SpecID", func(p issueops.IssuePatch) bool { return p.SpecID.Set }}, {"AwaitID", "W-IssuePatch.AwaitID", func(p issueops.IssuePatch) bool { return p.AwaitID.Set }}, diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index 970ca76de..8f6696f1d 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -719,8 +719,8 @@ func writeSideRows() []Row { patchRow("Owner", "an owner edit refuses ON updateIssue", "IssuePatchBody excludes owner — which the document itself calls an accident of order rather than a decision, since ApplyPatchBody DOES publish it and issues:batchApply's update item carries it"), patchRow("ClosedBySession", "a closed-by-session edit refuses", "neither IssuePatchBody nor ApplyPatchBody publishes closed_by_session"), - patchRow("Persistence", "a persistence-mode edit refuses", - "neither body publishes persistence: moving a row between planes mid-plan is a different act from writing its fields"), + patchRow("Persistence", "a persistence-mode edit refuses ON issues:batchApply", + "ApplyPatchBody excludes persistence: moving a row between planes mid-plan is a different act from writing its fields. IssuePatchBody publishes it behind issues.update.persistence, and updateIssue's encoder carries it (refusing before the dial against a server without that token)"), { ID: "W-IssuePatch.ParentID", Kind: KindRetired, What: "a parent re-hang through update used to refuse", diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index ea0df8554..9b98d3389 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -335,6 +335,9 @@ func (l *httpLifecycle) Update(ctx context.Context, req issueops.UpdateRequest) return issueops.UpdateResult{}, invalid("update names no field to write") } + if err := l.requirePersistenceMove(ctx, req); err != nil { + return issueops.UpdateResult{}, err + } req, err = l.applyTemplateGuardForServer(ctx, req) if err != nil { return issueops.UpdateResult{}, err @@ -417,6 +420,33 @@ func serverPredatesUpdateClaim(err error) bool { problem.Param == "claim" } +// requirePersistenceMove refuses, before anything is dialed, an update that +// moves the row between planes (IssuePatch.Persistence: `bd promote`, `bd +// update --persistent`) against a server that does not advertise +// wire.CapIssuesUpdatePersistence. Such a server refuses `patch.persistence` +// as an unknown parameter, and the move is one transaction on the server, so +// there is no client-side fallback: the refusal is the typed capability error +// naming the token (*wire.CapabilityError, which unwraps to +// *storage.ErrUnsupported with Capability set), never a dropped member and a +// "no change" answer. +func (l *httpLifecycle) requirePersistenceMove(ctx context.Context, req issueops.UpdateRequest) error { + if !req.Patch.Persistence.Set { + return nil + } + snap, err := l.store.snapshot(ctx) + if err != nil { + return err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapIssuesUpdatePersistence) { + return nil + } + // The case-2 text names the server and the token, so `bd promote` + // against an older server says what to upgrade rather than only that the + // operation is unsupported. It unwraps to *storage.ErrUnsupported with + // Capability set, like every other capability refusal. + return wire.NewCapabilityError(wire.OpUpdateIssue, wire.CapIssuesUpdatePersistence, l.store.target.String(), snap) +} + // applyTemplateGuardForServer decides, BEFORE the dial and from the cached // handshake, how the template read-only guard (storage // issueops.AuthorizeTemplateUpdate) reaches the server this request goes to. @@ -779,6 +809,12 @@ func encodeIssuePatch(patch issueops.IssuePatch) (map[string]any, error) { } setString(out, "assignee", patch.Assignee) setString(out, "parent_id", patch.ParentID) + // The plane move (issues.update.persistence). Update has already refused + // it, before the dial, against a server that does not advertise the token; + // this only spells it. + if patch.Persistence.Set { + out["persistence"] = string(patch.Persistence.Value) + } if metadataPatched(patch.Metadata) { document, err := encodeMetadataPatch(patch.Metadata) if err != nil { @@ -910,8 +946,6 @@ func refuseExcludedPatchMembers(patch issueops.IssuePatch) error { return refuse(encode.OpUpdateIssue, "W-IssuePatch.SpecID") case patch.AwaitID.Set: return refuse(encode.OpUpdateIssue, "W-IssuePatch.AwaitID") - case patch.Persistence.Set: - return refuse(encode.OpUpdateIssue, "W-IssuePatch.Persistence") } return nil } diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go index 08e5bc810..3362814c5 100644 --- a/internal/httpclient/served_claim_lifecycle_test.go +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -534,11 +534,12 @@ func TestServedLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t *test // `force_notes_overwrite` before them. Their rows (W-UpdateRequest.Claim, // .ForceAssigneeTransfer, .ForceClosePolicy) are RETIRED. // -// What still parks after them is TWO members updateIssue publishes nothing for -// (W-IssuePatch.Persistence, W-UpdateRequest.Provenance), each named so the -// lock counts the contract rather than reading it as unwritten. Both refusals -// are themselves asserted by TestUpdateRefusesEveryMemberTheWireExcludes, which -// RUNS, so no park here stands in for an unpinned behavior. +// What still parks after them is ONE member updateIssue publishes nothing for +// (W-UpdateRequest.Provenance), named so the lock counts the contract rather +// than reading it as unwritten. W-IssuePatch.Persistence left the parks with +// issues.update.persistence (S21b). The remaining refusal is itself asserted by +// TestUpdateRefusesEveryMemberTheWireExcludes, which RUNS, so no park here +// stands in for an unpinned behavior. // TestServedLifecycleUpdateClaimIsAMutationWhenThePatchRestoresTheRow is the // claim folded into an update: a mutation even when the patch beside it restores @@ -808,18 +809,28 @@ func TestServedLifecycleUpdateClosePolicy(t *testing.T) { conformance.RunLifecycleUpdateClosePolicy(t, t.Context(), newServedUpdateFixture(t, "hlcp")) } -// TestServedLifecycleUpdatePersistentPreservesUnversionedClass parks on -// IssuePatch.Persistence, which neither IssuePatchBody nor ApplyPatchBody -// publishes: moving a row between planes mid-plan is a different act from -// writing its fields, and a dropped persistence mode would answer "no change" -// to a request that asked for one. +// TestServedLifecycleUpdatePersistentPreservesUnversionedClass RUNS since +// issues.update.persistence (S21b): IssuePatchBody publishes `persistence` and +// the client sends it, so the restatement is the server's own role call. func TestServedLifecycleUpdatePersistentPreservesUnversionedClass(t *testing.T) { - skipKnownDivergence(t, "W-IssuePatch.Persistence", parkBead, - "the case restates an unversioned row as persistent, and the wire publishes no persistence member "+ - "on either patch body; the client refuses it rather than reporting a no-op it did not ask the server for") conformance.RunLifecycleUpdatePersistentPreservesUnversionedClass(t, t.Context(), newServedUpdateFixture(t, "hlpu")) } +// TestServedLifecycleUpdateWispModesRefuseAnUnversionedRow is the role's +// refusal answered as a 400 the client maps back to ErrValidation; before the +// role classified it, the server answered a 500 `internal` and emitted +// `request_error` for a request the caller could have fixed. +func TestServedLifecycleUpdateWispModesRefuseAnUnversionedRow(t *testing.T) { + conformance.RunLifecycleUpdateWispModesRefuseAnUnversionedRow(t, t.Context(), newServedUpdateFixture(t, "hluw")) +} + +// TestServedLifecycleUpdatePersistentPromotesAWisp is the plane move `bd +// promote` sends over http (issues.update.persistence, S21b), served by the +// role the direct route runs. +func TestServedLifecycleUpdatePersistentPromotesAWisp(t *testing.T) { + conformance.RunLifecycleUpdatePersistentPromotesAWisp(t, t.Context(), newServedUpdateFixture(t, "hlpw")) +} + // TestServedLifecycleUpdateProvenanceLabelsHistory is the update-side twin of // the reopen park above: the whole purpose of the field is the LABEL on the // history entry, and the server writes its own, so dropping it would leave the diff --git a/internal/httpclient/update_persistence_test.go b/internal/httpclient/update_persistence_test.go new file mode 100644 index 000000000..f53dadd7a --- /dev/null +++ b/internal/httpclient/update_persistence_test.go @@ -0,0 +1,85 @@ +package httpclient + +import ( + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// TestUpdatePersistenceIsSentOnlyToAServerThatAdvertisesIt pins the plane +// move `bd promote` (issueops.Promote) sends. Against a server advertising +// issues.update.persistence it is one updateIssue carrying +// `patch.persistence`; against one that does not, it is the typed capability +// refusal naming the token, raised BEFORE the dial: the member is never +// dropped (the server would answer "no change" to a move it never saw) and +// never sent to a server that would refuse it as an unknown parameter. +func TestUpdatePersistenceIsSentOnlyToAServerThatAdvertisesIt(t *testing.T) { + promote := issueops.UpdateRequest{ + Actor: "alice", IssueID: "bd-w1", + Patch: issueops.IssuePatch{Persistence: set(issueops.PersistenceModePersistent)}, + } + + t.Run("advertised", func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "7", Changed: true}} + lifecycle, err := New(testTarget(t), w, &apigen.ContextResponse{ + BdVersion: "1.2.3", + Capabilities: []string{wire.CapIssuesUpdateAllowTemplate, wire.CapIssuesUpdatePersistence}, + }).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + res, err := lifecycle.Update(t.Context(), promote) + if err != nil { + t.Fatalf("Update: %v", err) + } + if !res.Changed { + t.Error("Changed = false, want the server's answer") + } + if len(w.calls) != 1 || w.calls[0] != "updateIssue:bd-w1" { + t.Fatalf("calls = %v, want one updateIssue", w.calls) + } + if got := w.lastPatch["persistence"]; got != "persistent" { + t.Errorf("patch.persistence = %v, want \"persistent\"; patch = %v", got, w.lastPatch) + } + if len(w.lastPatch) != 1 { + t.Errorf("patch = %v, want the persistence member alone", w.lastPatch) + } + }) + + t.Run("unadvertised", func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "7"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + _, err = lifecycle.Update(t.Context(), promote) + requireCapabilityRefusal(t, err, wire.CapIssuesUpdatePersistence, w) + if !errors.Is(err, wire.ErrCapabilityAbsent) { + t.Errorf("err = %v, want it to match wire.ErrCapabilityAbsent", err) + } + // The message names the token, so a user knows what the server lacks. + if !strings.Contains(err.Error(), wire.CapIssuesUpdatePersistence) { + t.Errorf("refusal %q does not name %s", err.Error(), wire.CapIssuesUpdatePersistence) + } + }) + + t.Run("an update without the move is unaffected", func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "7"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + if _, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "alice", IssueID: "bd-1", Patch: issueops.IssuePatch{Title: set("t")}, + }); err != nil { + t.Fatalf("Update: %v", err) + } + if _, sent := w.lastPatch["persistence"]; sent { + t.Errorf("patch = %v carries persistence nobody asked for", w.lastPatch) + } + }) +} diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 5f7ee0131..bc395596c 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -249,6 +249,14 @@ const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" // (bd-jkp9v3). const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" +// CapIssuesUpdatePersistence is the behavior capability announcing that +// updateIssue accepts `patch.persistence` (issueops.IssuePatch.Persistence, +// the plane move behind `bd promote`). lifecycle.go sends the member only to +// a server advertising it, and refuses the move before the dial otherwise: +// there is no client-side fallback, because the move is one transaction on +// the server. +const CapIssuesUpdatePersistence = "issues.update.persistence" + // CapIssuesCreateDefaultPriority is the behavior capability announcing that // the server stores the create default priority for an ABSENT `priority` // member on issues.create, issues.batchCreate and issues.batchApply create @@ -408,7 +416,7 @@ var behaviorCapabilities = []string{ CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapCloseAutoCloseMolecule, CapListSearch, - CapListNoAssignee, CapListTitle, CapListIDs, + CapListNoAssignee, CapListTitle, CapListIDs, CapIssuesUpdatePersistence, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index aec04ef48..7dcbd2134 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -214,12 +214,12 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, httpapi.CapIssuesCloseAutoCloseMolecule, httpapi.CapIssuesListSearch, - httpapi.CapIssuesListNoAssignee, httpapi.CapIssuesListTitle, httpapi.CapListIDs, + httpapi.CapIssuesListNoAssignee, httpapi.CapIssuesListTitle, httpapi.CapListIDs, httpapi.CapIssuesUpdatePersistence, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit, "+ "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule, issues.list.search, "+ - "issues.list.noAssignee, issues.list.title and issues.list.ids tokens", behaviorCapabilities) + "issues.list.noAssignee, issues.list.title, issues.list.ids and issues.update.persistence tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 0532247bb..54ddf934f 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -318,6 +318,12 @@ func writeShapes() []writeShape { "Assignee": member("assignee"), "ParentID": member("parent_id"), "Metadata": member("metadata"), + // The plane move, carried since issues.update.persistence + // (S21b, `bd promote` over http). Only this shape: the batch + // item's ApplyPatchBody still excludes it, so + // W-IssuePatch.Persistence stays a refusal there — the Owner + // row's arrangement, the other way round. + "Persistence": member("persistence"), }, // NOTHING IS PENDING, and the set is empty rather than absent for the // reason the second list in accessors.go was: it held the two diff --git a/internal/httpclient/write_roles_test.go b/internal/httpclient/write_roles_test.go index f4645ef0a..b748bcb29 100644 --- a/internal/httpclient/write_roles_test.go +++ b/internal/httpclient/write_roles_test.go @@ -410,7 +410,9 @@ func TestUpdateRefusesEveryMemberTheWireExcludes(t *testing.T) { "Patch.ClosedBySession": {Patch: issueops.IssuePatch{ClosedBySession: set("s1")}}, "Patch.SpecID": {Patch: issueops.IssuePatch{SpecID: set("spec")}}, "Patch.AwaitID": {Patch: issueops.IssuePatch{AwaitID: set("await")}}, - "Patch.Persistence": {Patch: issueops.IssuePatch{Persistence: set(issueops.PersistenceModeEphemeral)}}, + // Patch.Persistence WAS here; issues.update.persistence publishes it + // (S21b). TestUpdatePersistenceIsSentOnlyToAServerThatAdvertisesIt + // is the positive assertion and the pre-dial capability refusal. // Patch.Labels.Add and Patch.Labels.Remove WERE here, and their absence // is the flip rather than an oversight: upstream #5510 published diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index d745aa427..8288bbb47 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -202,7 +202,8 @@ func (s *DoltStore) PartitionWispIDs(ctx context.Context, ids []string) (wispIDs } // PromoteFromEphemeral copies an issue from the wisps table to the issues table, -// clearing the Ephemeral flag. Used by bd promote and mol squash to crystallize wisps. +// clearing the Ephemeral flag. No bd command calls it: bd promote runs +// issueops.Promote, whose persistence move also carries the child counter. // // Uses direct SQL inserts to bypass IsEphemeralID routing, which would otherwise // redirect label/dependency/event writes back to wisp tables. diff --git a/internal/storage/dolt/issue_operations_persistence_test.go b/internal/storage/dolt/issue_operations_persistence_test.go index 3b4a8befe..a7c1ef504 100644 --- a/internal/storage/dolt/issue_operations_persistence_test.go +++ b/internal/storage/dolt/issue_operations_persistence_test.go @@ -3,8 +3,10 @@ package dolt import ( "context" "database/sql" + "errors" "testing" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -175,8 +177,39 @@ func TestMoveIssuePersistenceInTxRefusesUnversionedDemotion(t *testing.T) { _, err = issueops.MoveIssuePersistenceInTx(ctx, tx, current, types.PersistenceModeEphemeral, "test-actor") return err }) - if err == nil { - t.Fatal("unversioned demotion succeeded") + // The request is refused, not the row: ErrValidation is what makes this a + // 400 over http rather than a 500. + if !errors.Is(err, storage.ErrValidation) || !errors.Is(err, types.ErrUnversionedWispMove) { + t.Fatalf("unversioned demotion error = %v, want ErrValidation wrapping ErrUnversionedWispMove", err) + } + got, err := store.GetIssue(ctx, issue.ID) + if err != nil || got.Ephemeral || got.NoHistory { + t.Fatalf("refusal state = %#v, %v", got, err) + } +} + +func TestMoveIssuePersistenceInTxRefusesDemotionWithRetainedSnapshots(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx := context.Background() + issue := &types.Issue{ID: "persist-snapshot", Title: "snapshot", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatal(err) + } + // The snapshot tables have no default id since migration 0051. + if _, err := store.db.ExecContext(ctx, `INSERT INTO issue_snapshots (id, issue_id, snapshot_time, compaction_level, original_size, compressed_size, original_content) VALUES (?, ?, NOW(), 1, 1, 1, ?)`, "persist-snapshot-row", issue.ID, "snapshot"); err != nil { + t.Fatal(err) + } + err := store.withRetryTx(ctx, func(tx *sql.Tx) error { + current, err := issueops.GetIssueInTx(ctx, tx, issue.ID) + if err != nil { + return err + } + _, err = issueops.MoveIssuePersistenceInTx(ctx, tx, current, types.PersistenceModeEphemeral, "test-actor") + return err + }) + if !errors.Is(err, storage.ErrValidation) { + t.Fatalf("demotion with retained snapshots error = %v, want ErrValidation", err) } got, err := store.GetIssue(ctx, issue.ID) if err != nil || got.Ephemeral || got.NoHistory { diff --git a/internal/storage/dolt/lifecycle_update_contract_test.go b/internal/storage/dolt/lifecycle_update_contract_test.go index 2696c7834..0132cbd4f 100644 --- a/internal/storage/dolt/lifecycle_update_contract_test.go +++ b/internal/storage/dolt/lifecycle_update_contract_test.go @@ -87,6 +87,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("PersistentPreservesUnversionedClass", func(t *testing.T) { conformance.RunLifecycleUpdatePersistentPreservesUnversionedClass(t, ctx, fixture) }) + t.Run("WispModesRefuseAnUnversionedRow", func(t *testing.T) { + conformance.RunLifecycleUpdateWispModesRefuseAnUnversionedRow(t, ctx, fixture) + }) + t.Run("PersistentPromotesAWisp", func(t *testing.T) { + conformance.RunLifecycleUpdatePersistentPromotesAWisp(t, ctx, fixture) + }) t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 8c78e1ff5..0d6064dde 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -114,11 +114,12 @@ func (r *issueSQLRepositoryImpl) InsertBatch(ctx context.Context, issues []*type // PromoteFromEphemeral promotes an active wisp into the Dolt-versioned issues // plane in place: same id, wisp_type retained, labels/dependencies/events/ // comments carried across to the permanent tables, inbound wisp-targeted -// dependency edges retargeted, and blocked state recomputed. It delegates to -// the exact issueops implementation the classic (direct/embedded) route runs, -// so the two modes cannot drift. The issueops error is returned unwrapped on -// purpose: the CLI surfaces it verbatim ("wisp not found"), and that -// text is part of the classic error contract. +// dependency edges retargeted, and blocked state recomputed. It runs the +// issueops implementation DoltStorage.PromoteFromEphemeral runs and returns +// its error unwrapped ("wisp not found"). `bd promote` no longer reaches +// it on any route: issueops.Promote makes the persistence move instead +// (MovePersistence below, on this layer), which also carries the child +// counter. func (r *issueSQLRepositoryImpl) PromoteFromEphemeral(ctx context.Context, id, actor string) error { if id == "" { return errors.New("db: PromoteFromEphemeral: id must not be empty") diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index 7f5ec7208..716810eca 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -863,8 +863,10 @@ func (u *issueUseCaseImpl) isWispID(ctx context.Context, id string) (bool, error // PromoteWisp moves an active wisp to the Dolt-versioned issues plane, // preserving its id, wisp_type, labels, dependencies, events, and comments. // One-way: the promoted row is no longer ephemeral, so purge won't reclaim -// it. The repository error passes through unwrapped — the CLI surfaces it -// verbatim and its text is part of the classic error contract. +// it. The repository error passes through unwrapped. `bd promote` does not +// call it: the proxied-server route runs issueops.Promote, whose move is +// ApplyUpdate's persistence branch (MovePersistence), which also carries the +// child counter. func (u *issueUseCaseImpl) PromoteWisp(ctx context.Context, id, actor string) error { if id == "" { return fmt.Errorf("promote wisp: id must not be empty") diff --git a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go index 635c3ff13..93849a8ae 100644 --- a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go @@ -88,6 +88,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("PersistentPreservesUnversionedClass", func(t *testing.T) { conformance.RunLifecycleUpdatePersistentPreservesUnversionedClass(t, ctx, fixture) }) + t.Run("WispModesRefuseAnUnversionedRow", func(t *testing.T) { + conformance.RunLifecycleUpdateWispModesRefuseAnUnversionedRow(t, ctx, fixture) + }) + t.Run("PersistentPromotesAWisp", func(t *testing.T) { + conformance.RunLifecycleUpdatePersistentPromotesAWisp(t, ctx, fixture) + }) t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) diff --git a/internal/storage/issueops/persistence.go b/internal/storage/issueops/persistence.go index 1d8a1e95b..6a395c75c 100644 --- a/internal/storage/issueops/persistence.go +++ b/internal/storage/issueops/persistence.go @@ -2,8 +2,10 @@ package issueops import ( "context" + "errors" "fmt" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -32,6 +34,14 @@ func moveIssuePersistenceInTx(ctx context.Context, tx DBTX, current *types.Issue } ephemeral, noHistory, storageClass, err := types.NormalizePersistenceMode(*current, mode) if err != nil { + // The unversioned refusal is the request's: the row is sound and the + // move it asks for is one its class never makes. The other errors + // describe a row that breaks its own persistence invariants, so they + // stay unclassified; an invalid mode is refused before any update + // path gets here (ValidateUpdateRequest). + if errors.Is(err, types.ErrUnversionedWispMove) { + return PersistenceMoveResult{}, fmt.Errorf("%w: %w", storage.ErrValidation, err) + } return PersistenceMoveResult{}, err } sourceWisp, err := isActiveWispForPersistenceMove(ctx, tx, current.ID) @@ -164,6 +174,9 @@ func persistenceIssueTable(wisp bool) string { return "issues" } +// rejectPersistenceDemotion refuses a durable-to-wisp move of a row that has +// retained snapshots. The refusal is the request's (ErrValidation); a failed +// count is not. func rejectPersistenceDemotion(ctx context.Context, tx DBTX, id string) error { for _, table := range []string{"issue_snapshots", "compaction_snapshots"} { var count int @@ -171,7 +184,7 @@ func rejectPersistenceDemotion(ctx context.Context, tx DBTX, id string) error { return fmt.Errorf("check retained snapshots in %s: %w", table, err) } if count > 0 { - return fmt.Errorf("cannot demote issue %s with retained snapshots", id) + return fmt.Errorf("%w: cannot demote issue %s with retained snapshots", storage.ErrValidation, id) } } return nil diff --git a/internal/storage/uow/lifecycle_update_contract_test.go b/internal/storage/uow/lifecycle_update_contract_test.go index 85983bf4f..4a1a0677d 100644 --- a/internal/storage/uow/lifecycle_update_contract_test.go +++ b/internal/storage/uow/lifecycle_update_contract_test.go @@ -89,6 +89,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("PersistentPreservesUnversionedClass", func(t *testing.T) { conformance.RunLifecycleUpdatePersistentPreservesUnversionedClass(t, ctx, fixture) }) + t.Run("WispModesRefuseAnUnversionedRow", func(t *testing.T) { + conformance.RunLifecycleUpdateWispModesRefuseAnUnversionedRow(t, ctx, fixture) + }) + t.Run("PersistentPromotesAWisp", func(t *testing.T) { + conformance.RunLifecycleUpdatePersistentPromotesAWisp(t, ctx, fixture) + }) t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index d0bafd710..1d9ee1e0f 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -31,8 +31,8 @@ // # Where this plumbing deliberately differs from the DoltStorage one // // Every per-verb firing rule below is matched 1:1 against the decorator that -// implements it (each override cites its twin). These five are the exceptions, -// and they are listed here rather than argued in five places: +// implements it (each override cites its twin). These four are the exceptions, +// and they are listed here rather than argued in four places: // // 1. LABEL MULTIPLICITY ON CREATE. The DoltStorage chain replays a legacy // sequence for a create that carries labels: on_create with the labels @@ -61,16 +61,7 @@ // that writes nothing; the DoltStorage plumbing passes the precondition // into the close. See specWrites. // -// 5. THE PROMOTION COMMENT. `bd promote` records an audit comment on the -// promoted issue, and this plumbing's one comment verb fires on_update for -// it. The DoltStorage route writes that comment through the legacy -// AddComment, which the decorator wraps only INSIDE a transaction -// (hookTrackingTransaction.AddComment) and not at the store level, where -// promote calls it — so an embedded promote fires nothing. One verb cannot -// tell which caller it is serving, and "a comment landed" is the on_update -// this vocabulary has. -// -// A sixth is not a divergence but is worth stating: a unit of work whose commit +// A fifth is not a divergence but is worth stating: a unit of work whose commit // message is empty is ROLLED BACK by RunTxResult, and a rolled-back attempt // fires nothing — so a write that lands nothing reports nothing, on both // plumbings. diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 5114f2212..8626bf9d9 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -172,8 +172,9 @@ func TestRecordingIssueUseCaseCoversItsSurface(t *testing.T) { "ReclaimExpiredLeases": leases, "WakeExpiredDefers": "the deferred-work sweep is a clock crossing rather than a user's " + "mutation, and the DoltStorage plumbing fires nothing for it", - "PromoteWisp": "moves a row between planes without changing the bead; the DoltStorage " + - "plumbing has no promote hook either", + "PromoteWisp": "no route calls it: bd promote moves the row with ApplyUpdate's " + + "persistence spec, which records the update, and its DoltStorage twin, " + + "PromoteFromEphemeral, fires no hook either", }) } diff --git a/internal/types/storage_class_test.go b/internal/types/storage_class_test.go index d9d8dabcf..acaf36834 100644 --- a/internal/types/storage_class_test.go +++ b/internal/types/storage_class_test.go @@ -2,6 +2,8 @@ package types import ( "encoding/json" + "errors" + "fmt" "strings" "testing" "time" @@ -277,8 +279,16 @@ func TestNormalizePersistenceModeRejectsPlaneClassIncoherence(t *testing.T) { {"durable with explicit ephemeral", Issue{StorageClass: StorageClassEphemeral}, PersistenceModePersistent}, } for _, tc := range cases { - if _, _, _, err := NormalizePersistenceMode(tc.current, tc.target); err == nil { + _, _, _, err := NormalizePersistenceMode(tc.current, tc.target) + if err == nil { t.Errorf("%s: NormalizePersistenceMode succeeded, want error", tc.name) + continue + } + // A row that breaks its own invariants is not the request's refusal, + // even when its class is unversioned: callers classify only + // ErrUnversionedWispMove as validation. + if errors.Is(err, ErrUnversionedWispMove) { + t.Errorf("%s: NormalizePersistenceMode error %q matches ErrUnversionedWispMove, want the row-invariant error", tc.name, err) } } } @@ -294,8 +304,13 @@ func TestNormalizePersistenceModeRejectsUnsetAndUnknown(t *testing.T) { func TestNormalizePersistenceModeRejectsUnversionedDemotion(t *testing.T) { current := Issue{StorageClass: StorageClassUnversioned} for _, target := range []PersistenceMode{PersistenceModeEphemeral, PersistenceModeNoHistory} { - if _, _, _, err := NormalizePersistenceMode(current, target); err == nil { - t.Errorf("NormalizePersistenceMode(unversioned, %q) succeeded, want error", target) + _, _, _, err := NormalizePersistenceMode(current, target) + if !errors.Is(err, ErrUnversionedWispMove) { + t.Errorf("NormalizePersistenceMode(unversioned, %q) error = %v, want ErrUnversionedWispMove", target, err) + continue + } + if want := fmt.Sprintf("cannot move unversioned record to persistence mode %q", target); err.Error() != want { + t.Errorf("NormalizePersistenceMode(unversioned, %q) error = %q, want %q", target, err, want) } } } diff --git a/internal/types/types.go b/internal/types/types.go index 7a0edf045..b5461c910 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1017,12 +1017,19 @@ func (m PersistenceMode) IsValid() bool { return slices.Contains(validPersistenceModes, m) } +// ErrUnversionedWispMove is NormalizePersistenceMode's refusal to move an +// unversioned record to a wisp mode. The record is sound and the requested +// move is what is refused, so callers can classify it apart from the +// row-invariant errors the function also returns. +var ErrUnversionedWispMove = errors.New("cannot move unversioned record") + // NormalizePersistenceMode maps a requested mode from current to its exact // persistence fields without mutating an issue. The returned values are // Ephemeral, NoHistory, and StorageClass, respectively. Plane and retention // changes preserve the create-selected class. Promotion clears an explicit // ephemeral class marker to select normalized versioned storage. An -// unversioned record may remain persistent but cannot move to a wisp mode. +// unversioned record may remain persistent but cannot move to a wisp mode +// (ErrUnversionedWispMove). func NormalizePersistenceMode(current Issue, mode PersistenceMode) (bool, bool, StorageClass, error) { if !mode.IsValid() { return false, false, "", fmt.Errorf("invalid persistence mode %q", mode) @@ -1041,7 +1048,7 @@ func NormalizePersistenceMode(current Issue, mode PersistenceMode) (bool, bool, return false, false, "", fmt.Errorf("storage class ephemeral requires ephemeral or no_history") } if current.StorageClass == StorageClassUnversioned && mode != PersistenceModePersistent { - return false, false, "", fmt.Errorf("cannot move unversioned record to persistence mode %q", mode) + return false, false, "", fmt.Errorf("%w to persistence mode %q", ErrUnversionedWispMove, mode) } if persistenceMode(current) == mode { return current.Ephemeral, current.NoHistory, current.StorageClass, nil diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index e23cdd54a..62fec725e 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "molecule.go", "molecule_active.go", "moleculestepper.go", + "promote.go", "querier.go", "reader.go", "reader_ready_scope.go", @@ -63,6 +64,7 @@ go_test( "issueops_external_test.go", "molecule_internal_test.go", "molecule_test.go", + "promote_test.go", "search_test.go", ], embed = [":issueops"], diff --git a/issueops/promote.go b/issueops/promote.go new file mode 100644 index 000000000..d14d5bae0 --- /dev/null +++ b/issueops/promote.go @@ -0,0 +1,118 @@ +package issueops + +import ( + "context" + "fmt" +) + +// PromoteRequest describes one `bd promote`: move a wisp onto the durable +// issues plane and record why. +type PromoteRequest struct { + // Actor is the principal the move and the comment are attributed to. It + // must not be empty. + Actor string + // IssueID is the exact canonical id of the wisp. It must not be empty; + // fuzzy or prefix resolution is the caller's, for the reason + // GetRequest.ID gives. + IssueID string + // Reason is the optional text appended to the promotion comment. + Reason string + // Provenance labels the version-control history entry of the move, as + // UpdateRequest.Provenance describes. A remote backend labels its own + // entry, so a caller writing through one leaves it empty. + Provenance string +} + +// PromoteResult reports a promotion. +type PromoteResult struct { + // Issue is the promoted row as the move left it. + Issue *Issue + // CommentErr is set when the move succeeded but the promotion comment + // could not be written. The promotion stands: the comment is the audit + // note, not the act, and bd promote has always reported a failed comment + // as a warning rather than undoing the move. + CommentErr error +} + +// PromoteRoles are the three roles Promote composes. Every backend serves +// all three, so a promotion reads and writes the same way on every route. +type PromoteRoles struct { + Reader Reader + Lifecycle Lifecycle + Commenter Commenter +} + +// NotAWispError refuses a promotion of a row that is not an ephemeral wisp: +// it is already durable, or another writer promoted it first. A no-history +// wisp (NoHistory set, Ephemeral not) is refused with the same sentence, +// because the gate reads the Ephemeral flag as `bd promote` always has; +// `bd update --persistent` is the move for that row. +type NotAWispError struct { + ID string +} + +func (e *NotAWispError) Error() string { + return fmt.Sprintf("%s is not a wisp (already persistent)", e.ID) +} + +// PromotionComment is the audit comment a promotion records on the promoted +// row. Every route records exactly this text. +func PromotionComment(reason string) string { + comment := "Promoted from Level 0" + if reason != "" { + comment += ": " + reason + } + return comment +} + +// Promote is the one library entry for `bd promote`. It reads the row, +// refuses one that is not an ephemeral wisp (*NotAWispError), moves it to the +// durable plane with ONE Lifecycle.Update whose patch is the persistence move +// alone (Persistence = persistent: the row keeps its id, and its labels, +// dependencies, events and comments move with it, atomically, in the role's +// own transaction), and then records the promotion comment through the +// Commenter. +// +// A move that changed nothing lost a race to another promoter, and is +// reported as *NotAWispError without a second comment. Every refusal of the +// read or the move is returned unchanged, so a backend that cannot express +// the move (an http server without the issues.update.persistence capability) +// reaches the caller as its own typed refusal, before anything was written. +func Promote(ctx context.Context, roles PromoteRoles, req PromoteRequest) (PromoteResult, error) { + if req.Actor == "" { + return PromoteResult{}, fmt.Errorf("%w: promote: actor is required", ErrValidation) + } + if req.IssueID == "" { + return PromoteResult{}, fmt.Errorf("%w: promote: issue id is required", ErrValidation) + } + current, err := roles.Reader.Get(ctx, GetRequest{ID: req.IssueID}) + if err != nil { + return PromoteResult{}, err + } + if !current.Ephemeral { + return PromoteResult{}, &NotAWispError{ID: req.IssueID} + } + moved, err := roles.Lifecycle.Update(ctx, UpdateRequest{ + Actor: req.Actor, + IssueID: req.IssueID, + Patch: IssuePatch{ + Persistence: Field[PersistenceMode]{Set: true, Value: PersistenceModePersistent}, + }, + Provenance: req.Provenance, + }) + if err != nil { + return PromoteResult{}, err + } + if !moved.Changed { + return PromoteResult{}, &NotAWispError{ID: req.IssueID} + } + result := PromoteResult{Issue: moved.Issue} + if _, err := roles.Commenter.AddComment(ctx, AddCommentRequest{ + Author: req.Actor, + IssueID: req.IssueID, + Text: PromotionComment(req.Reason), + }); err != nil { + result.CommentErr = err + } + return result, nil +} diff --git a/issueops/promote_test.go b/issueops/promote_test.go new file mode 100644 index 000000000..28cb433cb --- /dev/null +++ b/issueops/promote_test.go @@ -0,0 +1,207 @@ +package issueops_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// promoteFakes is the three roles issueops.Promote composes. Each embeds its +// interface so a method the entry must not call panics on the nil. +type promoteFakes struct { + issue *issueops.Issue + getErr error + updateErr error + unchanged bool + commentErr error + + updates []issueops.UpdateRequest + comments []issueops.AddCommentRequest +} + +type promoteReader struct { + issueops.Reader + f *promoteFakes +} + +func (r promoteReader) Get(_ context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + if r.f.getErr != nil { + return nil, r.f.getErr + } + if r.f.issue == nil || r.f.issue.ID != req.ID { + return nil, issueops.ErrNotFound + } + return &issueops.IssueDetails{Issue: *r.f.issue}, nil +} + +type promoteLifecycle struct { + issueops.Lifecycle + f *promoteFakes +} + +func (l promoteLifecycle) Update(_ context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + l.f.updates = append(l.f.updates, req) + if l.f.updateErr != nil { + return issueops.UpdateResult{}, l.f.updateErr + } + promoted := *l.f.issue + promoted.Ephemeral = false + return issueops.UpdateResult{Issue: &promoted, Changed: !l.f.unchanged}, nil +} + +type promoteCommenter struct { + issueops.Commenter + f *promoteFakes +} + +func (c promoteCommenter) AddComment(_ context.Context, req issueops.AddCommentRequest) (issueops.AddCommentResult, error) { + c.f.comments = append(c.f.comments, req) + if c.f.commentErr != nil { + return issueops.AddCommentResult{}, c.f.commentErr + } + return issueops.AddCommentResult{Comment: &issueops.Comment{IssueID: req.IssueID, Author: req.Author, Text: req.Text}}, nil +} + +func (f *promoteFakes) roles() issueops.PromoteRoles { + return issueops.PromoteRoles{ + Reader: promoteReader{f: f}, + Lifecycle: promoteLifecycle{f: f}, + Commenter: promoteCommenter{f: f}, + } +} + +func wisp(id string) *issueops.Issue { + return &issueops.Issue{ID: id, Title: "a wisp", Ephemeral: true} +} + +func TestPromoteMovesTheWispThroughOnePersistenceUpdateThenComments(t *testing.T) { + f := &promoteFakes{issue: wisp("bd-w1")} + res, err := issueops.Promote(t.Context(), f.roles(), issueops.PromoteRequest{ + Actor: "alice", IssueID: "bd-w1", Reason: "keep it", Provenance: "bd: promote bd-w1", + }) + if err != nil { + t.Fatalf("Promote: %v", err) + } + if len(f.updates) != 1 { + t.Fatalf("updates = %d, want exactly one", len(f.updates)) + } + u := f.updates[0] + if u.Actor != "alice" || u.IssueID != "bd-w1" || u.Provenance != "bd: promote bd-w1" { + t.Errorf("update request = %+v, want actor alice, id bd-w1 and the caller's provenance", u) + } + if !u.Patch.Persistence.Set || u.Patch.Persistence.Value != issueops.PersistenceModePersistent { + t.Errorf("update patch persistence = %+v, want set to persistent", u.Patch.Persistence) + } + // The update is the plane move and nothing else. + only := issueops.IssuePatch{Persistence: u.Patch.Persistence} + if !reflect.DeepEqual(u.Patch, only) { + t.Errorf("update patch carries more than the persistence move: %+v", u.Patch) + } + if len(f.comments) != 1 { + t.Fatalf("comments = %d, want exactly one", len(f.comments)) + } + c := f.comments[0] + if c.Author != "alice" || c.IssueID != "bd-w1" || c.Text != "Promoted from Level 0: keep it" { + t.Errorf("comment = %+v, want alice's promotion comment with the reason", c) + } + if res.Issue == nil || res.Issue.Ephemeral || res.CommentErr != nil { + t.Errorf("result = %+v, want the promoted row and no comment error", res) + } +} + +func TestPromotionCommentWithoutAReason(t *testing.T) { + if got := issueops.PromotionComment(""); got != "Promoted from Level 0" { + t.Errorf("PromotionComment(\"\") = %q", got) + } +} + +func TestPromoteRefusesARowThatIsNotAWispBeforeWriting(t *testing.T) { + f := &promoteFakes{issue: &issueops.Issue{ID: "bd-1", Title: "durable"}} + _, err := issueops.Promote(t.Context(), f.roles(), issueops.PromoteRequest{Actor: "alice", IssueID: "bd-1"}) + var notWisp *issueops.NotAWispError + if !errors.As(err, ¬Wisp) || notWisp.ID != "bd-1" { + t.Fatalf("err = %v, want *NotAWispError for bd-1", err) + } + if err.Error() != "bd-1 is not a wisp (already persistent)" { + t.Errorf("message = %q, want the sentence bd promote always printed", err.Error()) + } + if len(f.updates) != 0 || len(f.comments) != 0 { + t.Errorf("wrote %d updates and %d comments, want none", len(f.updates), len(f.comments)) + } +} + +func TestPromoteReportsALostRaceAsNotAWisp(t *testing.T) { + // Another writer promoted the row between the read and the update: the + // update is a no-op, and no second promotion comment is written. + f := &promoteFakes{issue: wisp("bd-w2"), unchanged: true} + _, err := issueops.Promote(t.Context(), f.roles(), issueops.PromoteRequest{Actor: "alice", IssueID: "bd-w2"}) + var notWisp *issueops.NotAWispError + if !errors.As(err, ¬Wisp) { + t.Fatalf("err = %v, want *NotAWispError", err) + } + if len(f.comments) != 0 { + t.Errorf("wrote %d comments after a no-op promote, want none", len(f.comments)) + } +} + +func TestPromotePassesRefusalsThroughUnchanged(t *testing.T) { + refusal := &issueops.ErrUnsupported{Op: "Lifecycle.Update", Backend: "http", Capability: "issues.update.persistence"} + for name, f := range map[string]*promoteFakes{ + "missing": {}, + "get": {issue: wisp("bd-w3"), getErr: errors.New("boom")}, + "update": {issue: wisp("bd-w3"), updateErr: refusal}, + } { + t.Run(name, func(t *testing.T) { + _, err := issueops.Promote(t.Context(), f.roles(), issueops.PromoteRequest{Actor: "alice", IssueID: "bd-w3"}) + if err == nil { + t.Fatal("no error") + } + switch name { + case "missing": + if !errors.Is(err, issueops.ErrNotFound) { + t.Errorf("err = %v, want ErrNotFound", err) + } + case "update": + var unsup *issueops.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Capability != "issues.update.persistence" { + t.Errorf("err = %v, want the capability refusal unchanged", err) + } + } + if len(f.comments) != 0 { + t.Errorf("commented after a failed promote") + } + }) + } +} + +func TestPromoteKeepsThePromotionWhenTheCommentFails(t *testing.T) { + f := &promoteFakes{issue: wisp("bd-w4"), commentErr: errors.New("comment down")} + res, err := issueops.Promote(t.Context(), f.roles(), issueops.PromoteRequest{Actor: "alice", IssueID: "bd-w4"}) + if err != nil { + t.Fatalf("Promote: %v, want the promotion to stand", err) + } + if res.CommentErr == nil || res.Issue == nil { + t.Errorf("result = %+v, want the promoted row and the comment error", res) + } +} + +func TestPromoteValidatesTheRequest(t *testing.T) { + for name, req := range map[string]issueops.PromoteRequest{ + "no actor": {IssueID: "bd-w5"}, + "no id": {Actor: "alice"}, + } { + t.Run(name, func(t *testing.T) { + f := &promoteFakes{issue: wisp("bd-w5")} + _, err := issueops.Promote(t.Context(), f.roles(), req) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("err = %v, want ErrValidation", err) + } + if len(f.updates) != 0 { + t.Errorf("updated on an invalid request") + } + }) + } +} diff --git a/test/conformance/e2e_test.go b/test/conformance/e2e_test.go index dac3549d9..7ff80dc13 100644 --- a/test/conformance/e2e_test.go +++ b/test/conformance/e2e_test.go @@ -91,7 +91,7 @@ var corpus = []scenario{ {"create", "st", "--id", "cf-st1", "-t", "task"}, {"stale", "--days", "1", "--json"}, }}, - // Promote a wisp to a durable issue (PromoteFromEphemeral) — CLI-level differential + // Promote a wisp to a durable issue (issueops.Promote) — CLI-level differential // for one of the backfilled writes. {name: "promote", steps: [][]string{ {"create", "w", "--id", "cf-pw1", "-t", "task", "--ephemeral"}, From 70887e708abc85bee8c691955c4b1d985bb6188e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 14 Oct 2026 08:46:13 +0000 Subject: [PATCH 07/76] feat(http): serve IsBlockedBatch through the BlockingAnnotator role behind dependencies.blocking.isBlocked (#7552) * feat(http): serve IsBlockedBatch through the BlockingAnnotator role behind dependencies.blocking.isBlocked gc's native ready cache needs bd's stored, transitive is_blocked column for the active set (beads.BlockedQuerier.IsBlockedBatch). Over http the client answered it with the generated stub, so every reconcile logged `operation "IsBlockedBatch" not supported by the http backend`, the projection stayed disabled and native Ready took ~5 s on the MC shadow. No read a deployed (1.3.0) server serves can answer the column exactly: the /v0 reads hydrate issues without is_blocked, the blocking decoration is direct and two edge types only, and the flag also propagates down parent-child edges and reads waits-for / conditional-blocks gates. So this is an additive server behaviour plus a token, not a client-side composition. - issueops: BlockingRequest.IncludeIsBlocked asks for, and IssueBlocking.IsBlocked carries, each answered id's stored flag (nil = no row, never "free"). issueops.BlockedFlags is the one library entry for the batched read over the role, chunked at BlockedFlagsChunk (100, held equal to the served issue_id bound by an httpapi test). - Bodies: storage/issueops.ExecuteBlockingAnnotation reads the flag with IsBlockedBatchInTx in the same read transaction (dolt, embeddeddolt and every sqlkit-style store that calls it); the unit-of-work body asks the new DependencyUseCase/Repository IsBlockedBatch (same shared SQL body). externaldeps keeps IsBlockedBatch's external-blocker invariant on the local and remote annotators, and its Store.IsBlockedBatch no longer makes the raw edge read over a role-served store. - Wire: GET /v0/beads/dependencies/blocking accepts `is_blocked` behind the new behaviour token dependencies.blocking.isBlocked (spec, apigen, wire-shape golden). wire_revision stays 2. - Client: httpBlockingAnnotator forwards IncludeIsBlocked and refuses it BEFORE dialing with *ErrUnsupported{Capability: dependencies.blocking.isBlocked} against a server that does not advertise it; Store.IsBlockedBatch is issueops.BlockedFlags over that annotator, so gc's errors.As latch still sees the typed refusal on an old server, now naming the token that lifts it. Tests: conformance case RunBlockingAnnotatorReportsTheTransitiveBlockedFlag (child of a blocked parent is blocked, free is false, a ghost has no flag, unasked carries none, BlockedFlags agrees) on the dolt, embedded, uow and served legs; httpapi handler and chunk-bound tests; client tests for chunking, the wire shape and the zero-dial refusal; vocabulary/capability gates. Co-Authored-By: Claude Opus 5.5 * fix(externaldeps): keep the external blocker on the unit-of-work IsBlockedBatch bd serve on a server or proxied-server workspace serves through the unit-of-work arm (WrapUOWProvider) and advertises policy.external_dependencies, so a client trusts the server's answers and adds no external term of its own. That arm's dependency use case did not override IsBlockedBatch, so the new dependencies.blocking.isBlocked flag answered the stored is_blocked column, which has no external term. An issue blocked only by an unsatisfied external: ref came back is_blocked=false, with that ref listed in its own blocked_by, while GET /v0/beads/ready excluded it. dependencyUseCase.IsBlockedBatch now raises every id that has an unsatisfied external blocker, deciding by the edges and resolver GetReadyWork excludes by, read through the same unit of work. Unlike Ready it resolves only the refs of the ids the column answered: a client asks in chunks, and no chunk should reopen every foreign project the workspace names. An id naming no row stays absent. Also correct the store decorator's IsBlockedBatch doc (the served flag now comes from the annotator role), add the CHANGELOG entry for the new capability, and pin the flag against Ready on every arm: WrapUOWProvider, directly and through its BlockingAnnotator (an absent id stays absent), the local annotator, and the served test, which now asserts the flag and the batched read. A counting opener pins that the unit-of-work read opens only the foreign projects its asked rows name. Checked against a real bd serve on a proxied-server workspace. At 97bcb57738 an issue blocked by external:remote:payments answered is_blocked=false while /ready excluded it. With this change it answers true, and once the capability ships /ready includes the issue and is_blocked is false. Asked about a free issue alone, the server resolves no external project; asked about the remote-blocked issue, it resolves remote and leaves alone the unconfigured project another issue names. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * test(externaldeps): pin that a satisfied external blocker leaves the unit-of-work is_blocked false Review of 192b73676 (bee-ghosttrack): the two new IsBlockedBatch tests only had unsatisfied external refs, so a body that raised every answered id with any external record (ignoring the resolver) stayed green. TestWrapUOWProviderBlockedFlagResolvesOnlyTheAskedRefs's opener now returns a foreign store whose labelled index carries a CLOSED provider of provides:payments (the decorator_test.go shape). be-shipped blocks on external:remote:payments (satisfied) and must read false; be-blocked moves to external:remote:invoices (still unsatisfied) and reads true; both still open /projects/remote once. With the reviewer's mutant applied (`if _, ok := answered[id]; ok && len(state.refsByIssue) >= 0 {`) the test fails on be-shipped in both new cases; restored, it passes (bazel test //internal/storage/externaldeps:externaldeps_test). Co-Authored-By: Claude Opus 5.5 * test(externaldeps): pin the store arm's satisfied external ref and the unasked is_blocked flag The store arm decides an external blocker in two places: IsBlockedBatch's local loop skips a ref the resolver reports satisfied, and the BlockingAnnotator role folds IsBlocked's verdict, which applies the same filter, into the served flag. Both annotators, store and remote, add that flag only when the request asks for it. No test checked either store answer for a ref whose project had shipped the capability, or that an unasked annotation carries no flag, so dropping any of those checks left the package green. Pin them: IsBlockedBatch and the store annotator answer false for a row whose only blocker is external:remote:payments once remote has closed its provides:payments issue, and neither annotator flags a row unasked, even one an external ref blocks. This is the store half of bd-yewju3; the previous commit pinned the unit-of-work half. Each mutant below survives the package at the previous commit and fails exactly one test here (go test -tags=gms_pure_go ./internal/storage/externaldeps/; unmutated, bazel test //internal/storage/externaldeps:externaldeps_test passes): - decorator.go IsBlockedBatch: `&& !satisfied[dep.DependsOnID]` made `&& len(satisfied) >= 0` fails TestIsBlockedBatchHonorsShippedExternalRef. - decorator.go IsBlocked: `if !satisfied[ref.raw] {` made always true fails TestBlockingAnnotatorLeavesShippedExternalRefUnblocked. - role_accessors.go: `if req.IncludeIsBlocked {` made `if true {` fails TestBlockingAnnotatorAddsNoFlagUnasked. - remote_roles.go: dropping `req.IncludeIsBlocked &&` fails TestRemoteBlockingAnnotatorMergesExternalBlockers. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ .../blocking_annotator_contract.go | 62 +++++++ backend/conformance/role_bundle_cases.go | 1 + internal/httpapi/apigen/types.gen.go | 5 +- internal/httpapi/blocking.go | 5 +- internal/httpapi/blocking_test.go | 44 +++++ internal/httpapi/routes.go | 11 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 35 +++- .../httpapi/wireshape/testdata/golden.json | 15 ++ internal/httpclient/BUILD.bazel | 3 + internal/httpclient/blocked_querier.go | 22 +++ internal/httpclient/blocked_querier_test.go | 115 ++++++++++++ internal/httpclient/completeness_test.go | 1 - internal/httpclient/roles_graph.go | 13 ++ .../served_external_dependency_policy_test.go | 28 +++ internal/httpclient/served_graph_aux_test.go | 1 + internal/httpclient/unsupported_gen.go | 7 +- internal/httpclient/wire/handshake.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 7 +- .../dolt/blocking_annotator_contract_test.go | 3 + internal/storage/domain/db/dependency.go | 11 ++ internal/storage/domain/dependency.go | 18 ++ .../blocking_annotator_contract_test.go | 3 + internal/storage/externaldeps/decorator.go | 13 +- .../storage/externaldeps/decorator_test.go | 111 ++++++++++++ internal/storage/externaldeps/remote_roles.go | 25 +++ .../remote_roles_composition_test.go | 8 +- .../storage/externaldeps/role_accessors.go | 7 + .../storage/externaldeps/uow_decorator.go | 35 ++++ .../externaldeps/uow_decorator_test.go | 167 ++++++++++++++++++ .../storage/issueops/blocking_annotation.go | 25 ++- .../uow/blocking_annotator_contract_test.go | 3 + internal/storage/uow/blockingannotator.go | 10 +- internal/storage/uow/notifying_parity_test.go | 1 + issueops/blockingannotator.go | 71 ++++++++ 36 files changed, 887 insertions(+), 23 deletions(-) create mode 100644 internal/httpclient/blocked_querier.go create mode 100644 internal/httpclient/blocked_querier_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 241e92c95..abb76e98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `GET /v0/beads/dependencies/blocking` accepts `is_blocked`, behind the new + `dependencies.blocking.isBlocked` capability token (additive; no + wire_revision bump), and answers each item's stored, transitive + `is_blocked` flag, the column `bd ready` filters on; an item whose id names + no row carries none. `issueops.BlockingRequest` gains `IncludeIsBlocked` on + every BlockingAnnotator implementation, and `issueops.BlockedFlags` is the + batched read over it. The HTTP backend's `IsBlockedBatch` now serves + through it; against a server that does not advertise the token it still + refuses before dialing, now with a typed capability error naming the + token. An unsatisfied `external:` blocker sets the flag on every route, + just as it keeps the issue out of `bd ready`. - `bd promote` works over a connected HTTP workspace (S21b). Its one library entry, `issueops.Promote(ctx, PromoteRoles{Reader, Lifecycle, Commenter}, PromoteRequest)`, reads the row, refuses one that is not a wisp diff --git a/backend/conformance/blocking_annotator_contract.go b/backend/conformance/blocking_annotator_contract.go index 523b05037..e17b49fb9 100644 --- a/backend/conformance/blocking_annotator_contract.go +++ b/backend/conformance/blocking_annotator_contract.go @@ -546,3 +546,65 @@ func assertParent(t *testing.T, result publicops.BlockingResult, id, want string t.Errorf("entry %s parent = %q, want %q", id, got, want) } } + +// RunBlockingAnnotatorReportsTheTransitiveBlockedFlag pins +// BlockingRequest.IncludeIsBlocked and IssueBlocking.IsBlocked: asked, each id +// with a row carries its STORED TRANSITIVE is_blocked flag — a child of a +// blocked parent is blocked with no blocking edge of its own, which BlockedBy +// alone would read as free — and an id naming no row carries none. Unasked, +// no item carries a flag. This is the read issueops.BlockedFlags (storage's +// IsBlockedBatch on a role-served backend) is built on. +func RunBlockingAnnotatorReportsTheTransitiveBlockedFlag(t *testing.T, ctx context.Context, fixture BlockingAnnotatorFixture) { + t.Helper() + blocker := fixture.IssuePrefix + "-flag-blocker" + blocked := fixture.IssuePrefix + "-flag-blocked" + child := fixture.IssuePrefix + "-flag-child" + free := fixture.IssuePrefix + "-flag-free" + ghost := fixture.IssuePrefix + "-flag-ghost" + seedBlockingIssues(t, ctx, fixture, blocker, blocked, child, free) + seedBlockingEdge(t, ctx, fixture, child, blocked, types.DepParentChild) + seedBlockingEdge(t, ctx, fixture, blocked, blocker, types.DepBlocks) + + ids := []string{blocked, child, free, ghost} + unasked := annotateBlocking(t, ctx, fixture, publicops.BlockingRequest{IDs: ids}) + for _, item := range unasked.Items { + if item.IsBlocked != nil { + t.Errorf("unasked: %s carries IsBlocked=%v, want no flag", item.ID, *item.IsBlocked) + } + } + + asked := annotateBlocking(t, ctx, fixture, publicops.BlockingRequest{IDs: ids, IncludeIsBlocked: true}) + want := map[string]*bool{blocked: new(true), child: new(true), free: new(false), ghost: nil} + if len(asked.Items) != len(ids) { + t.Fatalf("asked: %d items, want %d", len(asked.Items), len(ids)) + } + for _, item := range asked.Items { + w, ok := want[item.ID] + if !ok { + t.Errorf("asked: unexpected item %s", item.ID) + continue + } + switch { + case w == nil && item.IsBlocked != nil: + t.Errorf("asked: %s (no row) IsBlocked=%v, want no flag", item.ID, *item.IsBlocked) + case w != nil && item.IsBlocked == nil: + t.Errorf("asked: %s IsBlocked absent, want %v", item.ID, *w) + case w != nil && *item.IsBlocked != *w: + t.Errorf("asked: %s IsBlocked=%v, want %v", item.ID, *item.IsBlocked, *w) + } + } + + flags, err := publicops.BlockedFlags(ctx, fixture.Annotator, append(ids, blocked, "")) + if err != nil { + t.Fatalf("BlockedFlags: %v", err) + } + wantFlags := map[string]bool{blocked: true, child: true, free: false} + if len(flags) != len(wantFlags) { + t.Errorf("BlockedFlags = %v, want %v (no entry for the ghost or the blank id)", flags, wantFlags) + } + for id, w := range wantFlags { + if got, ok := flags[id]; !ok || got != w { + t.Errorf("BlockedFlags[%s] = %v (present %v), want %v", id, got, ok, w) + } + } +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index caaa5dc7a..ff52c67e4 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -127,6 +127,7 @@ var roleContractCases = []roleContract{ RunBlockingAnnotatorRefusesAnEmptyID, RunBlockingAnnotatorLeavesTheRequestAlone, RunBlockingAnnotatorWritesNothing, + RunBlockingAnnotatorReportsTheTransitiveBlockedFlag, ), // One contract file covers both halves of workspace identity: the eight diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index f0f16b608..8cdf6e633 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1101,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, and `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`, and `dependencies.blocking.isBlocked`, which announces that `GET /v0/beads/dependencies/blocking` accepts `is_blocked` and answers each item's stored, transitive `is_blocked` flag (see that parameter) rather than silently answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2261,6 +2261,9 @@ type ListBlockingAnnotationsParams struct { // // Repeats collapse: an id named twice is one entry, at the position of its first mention. IssueId []string `form:"issue_id" json:"issue_id"` + + // IsBlocked `true` adds each item's `is_blocked`: the issue's stored, TRANSITIVE readiness flag — the column the ready set filters on, which a child inherits from a blocked parent and which `waits-for` and `conditional-blocks` gates set — rather than the two-edge-type, direct `blocked_by` summary. It is the batched readiness projection a client cache needs to answer readiness with the server's own predicate. An id that names no row in either plane gets no `is_blocked` member. Absent or `false` answers exactly as before. Behind the `dependencies.blocking.isBlocked` capability token; a server that predates it answers `400` with `param: "is_blocked"` and `reason: "unknown_parameter"`. + IsBlocked *bool `form:"is_blocked,omitempty" json:"is_blocked,omitempty"` } // GetDependencyTreeParams defines parameters for GetDependencyTree. diff --git a/internal/httpapi/blocking.go b/internal/httpapi/blocking.go index 62ceb0c03..03608f528 100644 --- a/internal/httpapi/blocking.go +++ b/internal/httpapi/blocking.go @@ -27,6 +27,9 @@ func (s *Server) handleBlockingAnnotations(w http.ResponseWriter, r *http.Reques q := newQuery(r.URL.Query()) ids := q.list("issue_id") + // is_blocked rides CapDependenciesBlockingIsBlocked: the role's + // IncludeIsBlocked, each item's stored transitive flag. + includeIsBlocked := q.boolean("is_blocked") if !s.acceptQuery(w, r, q) { return } @@ -51,7 +54,7 @@ func (s *Server) handleBlockingAnnotations(w http.ResponseWriter, r *http.Reques s.failErr(w, r, err) return } - result, err := annotator.AnnotateBlocking(r.Context(), issueops.BlockingRequest{IDs: ids}) + result, err := annotator.AnnotateBlocking(r.Context(), issueops.BlockingRequest{IDs: ids, IncludeIsBlocked: includeIsBlocked}) if err != nil { s.failBlockingErr(w, r, err) return diff --git a/internal/httpapi/blocking_test.go b/internal/httpapi/blocking_test.go index 2dfd38822..a576d41d7 100644 --- a/internal/httpapi/blocking_test.go +++ b/internal/httpapi/blocking_test.go @@ -175,3 +175,47 @@ func TestABlockingRoleRefusalIsTheDocumentedBadRequest(t *testing.T) { t.Errorf("status = %d for an opaque role failure, want 500: %s", resp.StatusCode, readAll(t, resp)) } } + +// TestBlockingAnnotationsCarryTheTransitiveFlagWhenAsked pins the +// dependencies.blocking.isBlocked parameter: `is_blocked=true` reaches the +// role as IncludeIsBlocked, the role's flag is shipped as `is_blocked`, an +// item without one omits the member, and a request without the parameter +// asks for nothing. +func TestBlockingAnnotationsCarryTheTransitiveFlagWhenAsked(t *testing.T) { + blocked := true + annotator := &roleBlockingAnnotator{result: issueops.BlockingResult{Items: []issueops.IssueBlocking{ + {ID: "bd-1", BlockedBy: []string{}, Blocks: []string{}, IsBlocked: &blocked}, + {ID: "bd-ghost", BlockedBy: []string{}, Blocks: []string{}}, + }}} + ts := newBlockingServer(t, annotator) + + body := decodeBody(t, ts.get(t, "/v0/beads/dependencies/blocking?issue_id=bd-1&issue_id=bd-ghost&is_blocked=true")) + reads := annotator.blockingRequests() + if len(reads) != 1 || !reads[0].IncludeIsBlocked { + t.Fatalf("role saw %+v, want one request with IncludeIsBlocked", reads) + } + items, _ := body["items"].([]any) + if len(items) != 2 { + t.Fatalf("items = %v", body["items"]) + } + if first, _ := items[0].(map[string]any); first["is_blocked"] != true { + t.Errorf("items[0].is_blocked = %v, want true", first["is_blocked"]) + } + if second, _ := items[1].(map[string]any); second["is_blocked"] != nil { + t.Errorf("items[1] carries is_blocked %v; an item with no flag omits it", second["is_blocked"]) + } + + ts.get(t, "/v0/beads/dependencies/blocking?issue_id=bd-1") + if reads := annotator.blockingRequests(); len(reads) != 2 || reads[1].IncludeIsBlocked { + t.Errorf("a request without is_blocked asked for the flag: %+v", reads) + } +} + +// TestBlockedFlagsChunkIsTheServedAnchorBound holds issueops.BlockedFlagsChunk +// to this operation's issue_id bound, so the library entry never asks a +// question the served operation refuses. +func TestBlockedFlagsChunkIsTheServedAnchorBound(t *testing.T) { + if issueops.BlockedFlagsChunk != maxDependencyAnchors { + t.Errorf("issueops.BlockedFlagsChunk = %d, maxDependencyAnchors = %d; they must be equal", issueops.BlockedFlagsChunk, maxDependencyAnchors) + } +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index a724540e1..b70f2e6f1 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -166,6 +166,15 @@ const ( // (E9-S2), adopted unchanged so the two servers publish one `id`. const CapListIDs = "issues.list.ids" +// CapDependenciesBlockingIsBlocked is the behavior capability that advertises +// GET /v0/beads/dependencies/blocking's `is_blocked` parameter (blocking.go): +// issueops.BlockingRequest.IncludeIsBlocked, which adds each item's stored, +// transitive is_blocked flag to the decoration. It is the batched readiness +// projection behind storage's IsBlockedBatch on a role-served backend +// (issueops.BlockedFlags). The client refuses the read locally, naming this +// token, against a server that does not advertise it. +const CapDependenciesBlockingIsBlocked = "dependencies.blocking.isBlocked" + // CapIssuesSweepWispsPlane is the behavior capability that advertises the // `wisps-plane` value of SweepRequest.tier (sweep.go, spec SweepRequest.tier): // a third, wider tier that clears the whole wisps table plus any durable bead @@ -1044,7 +1053,7 @@ var behaviorCapabilities = []string{ CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapIssuesCloseAutoCloseMolecule, CapIssuesListSearch, - CapIssuesListNoAssignee, CapIssuesListTitle, CapListIDs, CapIssuesUpdatePersistence, + CapIssuesListNoAssignee, CapIssuesListTitle, CapListIDs, CapIssuesUpdatePersistence, CapDependenciesBlockingIsBlocked, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 43e983b2e..97899d2d1 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -639,7 +639,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", "issues.close.autoCloseMolecule", "issues.list.search", - "issues.list.noAssignee", "issues.list.title", "issues.list.ids", "issues.update.persistence") + "issues.list.noAssignee", "issues.list.title", "issues.list.ids", "issues.update.persistence", "dependencies.blocking.isBlocked") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 3556c5e99..e937cf752 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -5645,6 +5645,25 @@ paths: maxItems: 100 items: type: string + - name: is_blocked + in: query + description: >- + `true` adds each item's `is_blocked`: the issue's stored, + TRANSITIVE readiness flag — the column the ready set filters on, + which a child inherits from a blocked parent and which + `waits-for` and `conditional-blocks` gates set — rather than the + two-edge-type, direct `blocked_by` summary. It is the batched + readiness projection a client cache needs to answer readiness + with the server's own predicate. An id that names no row in + either plane gets no `is_blocked` member. Absent or `false` + answers exactly as before. Behind the + `dependencies.blocking.isBlocked` capability token; a server + that predates it answers `400` with `param: "is_blocked"` and + `reason: "unknown_parameter"`. + required: false + schema: + type: boolean + default: false security: - bearerToken: [] responses: @@ -8119,6 +8138,13 @@ components: carries several `parent-child` edges, which one appears here is not specified. A client that needs every structural edge reads them from `GET /v0/beads/dependencies`. + is_blocked: + type: boolean + description: >- + The issue's stored, transitive is_blocked flag — present only when + the request carried `is_blocked=true` (capability + `dependencies.blocking.isBlocked`) and the id names a row. Its + absence never means "not blocked". # --------------------------------------------------------------------- # Envelope schemas. Almost all of these have no canonical Go type behind @@ -8299,13 +8325,16 @@ components: member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering - `unknown_parameter`, and `issues.list.noAssignee`, + `unknown_parameter`, `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering - `unknown_parameter`. One - behavior token is + `unknown_parameter`, and `dependencies.blocking.isBlocked`, which + announces that `GET /v0/beads/dependencies/blocking` accepts + `is_blocked` and answers each item's stored, transitive + `is_blocked` flag (see that parameter) rather than silently + answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index b136b13af..c2d2882d3 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -2345,6 +2345,12 @@ "type": "string", "required": true }, + { + "schema": "IssueBlocking", + "member": "is_blocked", + "type": "boolean", + "required": false + }, { "schema": "IssueBlocking", "member": "parent", @@ -5667,6 +5673,15 @@ "explode": true, "default": "false" }, + { + "schema": "param:listBlockingAnnotations", + "member": "query:is_blocked", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, { "schema": "param:listBlockingAnnotations", "member": "query:issue_id", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 68803269c..95ee71185 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "batchcloser.go", "batchcreator.go", "batchgetter.go", + "blocked_querier.go", "blocked_recomputer.go", "bridge.go", "ca.go", @@ -96,6 +97,7 @@ go_test( srcs = [ "backend_credential_test.go", "batchapplier_test.go", + "blocked_querier_test.go", "brief_test.go", "bulk_roles_test.go", "ca_permissions_unix_test.go", @@ -194,6 +196,7 @@ go_test( embed = [":httpclient"], shard_count = 2, deps = [ + "//:beads", "//backend/conformance", "//internal/eventsjournal", "//internal/hooks", diff --git a/internal/httpclient/blocked_querier.go b/internal/httpclient/blocked_querier.go new file mode 100644 index 000000000..dea6fecda --- /dev/null +++ b/internal/httpclient/blocked_querier.go @@ -0,0 +1,22 @@ +package httpclient + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// IsBlockedBatch serves beads.BlockedQuerier's batched read — the stored, +// transitive is_blocked flag per id, an id naming no row absent — through the +// one library entry, issueops.BlockedFlags, over this store's +// BlockingAnnotator (listBlockingAnnotations with `is_blocked`, at most +// issueops.BlockedFlagsChunk ids per request). +// +// Against a server that does not advertise dependencies.blocking.isBlocked +// the annotator refuses before dialing with *ErrUnsupported naming the token, +// so a caller's errors.As on the typed refusal (gc's ready projection latches +// its degrade on it) sees the same sentinel the generated stub answered with +// before, now with the capability that would lift it. +func (s *Store) IsBlockedBatch(ctx context.Context, ids []string) (map[string]bool, error) { + return issueops.BlockedFlags(ctx, httpBlockingAnnotator{store: s}, ids) +} diff --git a/internal/httpclient/blocked_querier_test.go b/internal/httpclient/blocked_querier_test.go new file mode 100644 index 000000000..b37391375 --- /dev/null +++ b/internal/httpclient/blocked_querier_test.go @@ -0,0 +1,115 @@ +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "testing" + + beads "github.com/steveyegge/beads" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// TestIsBlockedBatchRidesTheBlockingAnnotatorBehindItsCapability pins the +// client half of the http IsBlockedBatch: beads.AsBlockedQuerier finds it, it +// asks listBlockingAnnotations with is_blocked=true, chunked at +// issueops.BlockedFlagsChunk ids, and answers id -> flag with an id the server +// gave no flag (no row) absent. +func TestIsBlockedBatchRidesTheBlockingAnnotatorBehindItsCapability(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{"dependencies.blocking", wire.CapBlockingIsBlocked}} + w := &countingWire{} + w.preflight = func(context.Context, string) error { return nil } + w.do = func(_ context.Context, req wire.Request, out any) error { + w.dialed = append(w.dialed, req) + items := make([]map[string]any, 0) + for _, id := range req.Query["issue_id"] { + item := map[string]any{"id": id, "blocked_by": []string{}, "blocks": []string{}} + switch id { + case "x-ghost": + case "x-7", "x-150": + item["is_blocked"] = true + default: + item["is_blocked"] = false + } + items = append(items, item) + } + raw, _ := json.Marshal(map[string]any{"items": items}) + return json.Unmarshal(raw, out) + } + var store beads.Storage = New(testTarget(t), w, served) + querier, ok := beads.AsBlockedQuerier(store) + if !ok { + t.Fatal("AsBlockedQuerier(http store) = false") + } + ids := []string{"x-ghost"} + for i := range 160 { + ids = append(ids, fmt.Sprintf("x-%d", i)) + } + ids = append(ids, "x-7") // a repeat is asked once + flags, err := querier.IsBlockedBatch(t.Context(), ids) + if err != nil { + t.Fatalf("IsBlockedBatch: %v", err) + } + if len(w.dialed) != 2 { + t.Fatalf("dialed %d requests, want 2 (161 distinct ids at %d per request)", len(w.dialed), issueops.BlockedFlagsChunk) + } + for _, req := range w.dialed { + if req.Op != wire.OpListBlockingAnnotations || req.Path != wire.PathDependenciesBlocking { + t.Errorf("dialed %s %s, want listBlockingAnnotations", req.Op, req.Path) + } + if got := req.Query["is_blocked"]; !slices.Equal(got, []string{"true"}) { + t.Errorf("query[is_blocked] = %v, want [true]", got) + } + if n := len(req.Query["issue_id"]); n > issueops.BlockedFlagsChunk { + t.Errorf("a request named %d ids, over the %d bound", n, issueops.BlockedFlagsChunk) + } + } + if len(flags) != 160 { + t.Errorf("len(flags) = %d, want 160 (the ghost absent)", len(flags)) + } + if _, ok := flags["x-ghost"]; ok { + t.Error("an id the server gave no flag is present; absent means no row") + } + if !flags["x-7"] || !flags["x-150"] || flags["x-8"] { + t.Errorf("flags x-7=%v x-150=%v x-8=%v, want true true false", flags["x-7"], flags["x-150"], flags["x-8"]) + } +} + +// TestIsBlockedBatchRefusesBeforeDialingWithoutTheCapability is the skew side: +// against a server that does not advertise dependencies.blocking.isBlocked (the +// deployed 1.3.0 serve, for one), the read refuses before the dial with the +// typed refusal gc's ready projection latches its degrade on, now naming the +// capability that would lift it. Sending it would cost a 400 +// unknown_parameter; dropping is_blocked would pass "no flag" off as "free". +func TestIsBlockedBatchRefusesBeforeDialingWithoutTheCapability(t *testing.T) { + masked := &apigen.ContextResponse{Capabilities: []string{"dependencies.blocking"}} + w := newCountingWire(`{"items":[]}`) + _, err := New(testTarget(t), w, masked).IsBlockedBatch(t.Context(), []string{"x-1"}) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("IsBlockedBatch error = %v, want *storage.ErrUnsupported", err) + } + if unsup.Capability != wire.CapBlockingIsBlocked { + t.Errorf("refusal capability = %q, want %s", unsup.Capability, wire.CapBlockingIsBlocked) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0", len(w.dialed)) + } + + // The plain decoration is unaffected by the missing token. + annotator, err := New(testTarget(t), w, masked).BlockingAnnotator() + if err != nil { + t.Fatal(err) + } + if _, err := annotator.AnnotateBlocking(t.Context(), issueops.BlockingRequest{IDs: []string{"x-1"}}); err != nil { + t.Fatalf("AnnotateBlocking without the flag: %v", err) + } + if w.query(t).Has("is_blocked") { + t.Error("a request without IncludeIsBlocked sent is_blocked") + } +} diff --git a/internal/httpclient/completeness_test.go b/internal/httpclient/completeness_test.go index a0c18d544..d8b06ba89 100644 --- a/internal/httpclient/completeness_test.go +++ b/internal/httpclient/completeness_test.go @@ -260,7 +260,6 @@ var legitimatelyUnsupported = map[string]string{ "GetDependentRecordsForIssues": "raw read: no multi-id dependents operation", "GetDependents": "raw read: no dependents operation off getIssue", "IsBlocked": "raw read: role BlockingAnnotator serves this shape", - "IsBlockedBatch": "raw read: role BlockingAnnotator serves this shape", "IterAllDependencyRecords": "raw read: streaming has no wire shape", "IterDependenciesWithMetadata": "raw read: streaming has no wire shape", "IterDependentsWithMetadata": "raw read: streaming has no wire shape", diff --git a/internal/httpclient/roles_graph.go b/internal/httpclient/roles_graph.go index 3a0dda6a5..487930a26 100644 --- a/internal/httpclient/roles_graph.go +++ b/internal/httpclient/roles_graph.go @@ -540,6 +540,19 @@ func (a httpBlockingAnnotator) AnnotateBlocking(ctx context.Context, req issueop for _, id := range ids { q.Add("issue_id", id) } + if req.IncludeIsBlocked { + // Refused before the dial against a server that does not advertise + // the parameter: it would answer unknown_parameter, and an answer + // without the flag must never pass for "nothing is blocked". + served, err := a.store.servesCapability(ctx, wire.CapBlockingIsBlocked) + if err != nil { + return issueops.BlockingResult{}, err + } + if !served { + return issueops.BlockingResult{}, a.store.unsupportedCapability("BlockingAnnotator.AnnotateBlocking", wire.CapBlockingIsBlocked) + } + q.Set("is_blocked", "true") + } var body apigen.BlockingAnnotations if err := a.store.dispatch(ctx, wire.Request{ diff --git a/internal/httpclient/served_external_dependency_policy_test.go b/internal/httpclient/served_external_dependency_policy_test.go index 3b11a08bd..896173cd6 100644 --- a/internal/httpclient/served_external_dependency_policy_test.go +++ b/internal/httpclient/served_external_dependency_policy_test.go @@ -5,7 +5,9 @@ package httpclient import ( "errors" "fmt" + "maps" "slices" + "strconv" "testing" "github.com/steveyegge/beads/internal/storage" @@ -284,6 +286,32 @@ func TestServedRolesKeepTheExternalDependencyPolicy(t *testing.T) { t.Errorf("AnnotateBlocking over http: %s blocked by %v", item.ID, item.BlockedBy) } } + // The stored is_blocked column has no external term, so the server calls + // the blocked issue unblocked; the flag and the batch read over it must + // still say blocked, as Ready does. + flagged, err := annotator.AnnotateBlocking(ctx, issueops.BlockingRequest{IDs: []string{blockedID, freeID}, IncludeIsBlocked: true}) + if err != nil { + t.Fatalf("AnnotateBlocking(IncludeIsBlocked) over http: %v", err) + } + if len(flagged.Items) != 2 { + t.Fatalf("AnnotateBlocking(IncludeIsBlocked) over http answered %d items, want 2", len(flagged.Items)) + } + for _, item := range flagged.Items { + got := "absent" + if item.IsBlocked != nil { + got = strconv.FormatBool(*item.IsBlocked) + } + if want := strconv.FormatBool(item.ID == blockedID); got != want { + t.Errorf("AnnotateBlocking(IncludeIsBlocked) over http: %s is_blocked = %s, want %s", item.ID, got, want) + } + } + flags, err := decorated.IsBlockedBatch(ctx, []string{blockedID, freeID}) + if err != nil { + t.Fatalf("IsBlockedBatch over http: %v", err) + } + if want := map[string]bool{blockedID: true, freeID: false}; !maps.Equal(flags, want) { + t.Errorf("IsBlockedBatch over http = %v, want %v", flags, want) + } walker, err := decorated.TreeWalker() if err != nil { diff --git a/internal/httpclient/served_graph_aux_test.go b/internal/httpclient/served_graph_aux_test.go index ce8ac1954..ce4d1506f 100644 --- a/internal/httpclient/served_graph_aux_test.go +++ b/internal/httpclient/served_graph_aux_test.go @@ -177,6 +177,7 @@ func TestServedBlockingAnnotatorContract(t *testing.T) { {"RefusesAnEmptyID", conformance.RunBlockingAnnotatorRefusesAnEmptyID}, {"LeavesTheRequestAlone", conformance.RunBlockingAnnotatorLeavesTheRequestAlone}, {"WritesNothing", conformance.RunBlockingAnnotatorWritesNothing}, + {"ReportsTheTransitiveBlockedFlag", conformance.RunBlockingAnnotatorReportsTheTransitiveBlockedFlag}, } { t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) } diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 3c72a4a97..7c900ccb7 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -427,11 +427,6 @@ func (unsupportedDoltStorage) IsBlocked(_ context.Context, _ string) (_ bool, _ return } -func (unsupportedDoltStorage) IsBlockedBatch(_ context.Context, _ []string) (_ map[string]bool, err error) { - err = errUnsupported("IsBlockedBatch") - return -} - func (unsupportedDoltStorage) IterAllDependencyRecords(_ context.Context) (_ storage.Iter[types.Dependency], err error) { err = errUnsupported("IterAllDependencyRecords") return @@ -732,4 +727,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (143 of 201 methods generated; 58 left to this package's hand-written set). +// NOTE: partial shell (142 of 201 methods generated; 59 left to this package's hand-written set). diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index bc395596c..c555d2e25 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -179,6 +179,15 @@ const ( CapListIDs = "issues.list.ids" ) +// CapBlockingIsBlocked is the behavior capability announcing that +// listBlockingAnnotations honors `is_blocked` (issueops.BlockingRequest. +// IncludeIsBlocked), spelled exactly as httpapi's +// CapDependenciesBlockingIsBlocked. The annotator refuses a request asking +// for the flag LOCALLY, with a typed capability error naming this token, when +// the server does not advertise it, so the IsBlockedBatch read never dials a +// server that would answer `unknown_parameter`. +const CapBlockingIsBlocked = "dependencies.blocking.isBlocked" + // CapBatchApplyLarge is the behavior capability announcing that issues.batchApply // accepts a batch larger than the compiled-in floor (100 items), up to the // raised ceiling (1000), spelled exactly as httpapi's constant of the same name @@ -416,7 +425,7 @@ var behaviorCapabilities = []string{ CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapCloseAutoCloseMolecule, CapListSearch, - CapListNoAssignee, CapListTitle, CapListIDs, CapIssuesUpdatePersistence, + CapListNoAssignee, CapListTitle, CapListIDs, CapIssuesUpdatePersistence, CapBlockingIsBlocked, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 7dcbd2134..5059fd660 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -160,6 +160,9 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapListIDs != httpapi.CapListIDs { t.Errorf("CapListIDs = %q, server says %q", CapListIDs, httpapi.CapListIDs) } + if CapBlockingIsBlocked != httpapi.CapDependenciesBlockingIsBlocked { + t.Errorf("CapBlockingIsBlocked = %q, server says %q", CapBlockingIsBlocked, httpapi.CapDependenciesBlockingIsBlocked) + } if CapCountScope != httpapi.CapIssuesCountScope { t.Errorf("CapCountScope = %q, server says %q", CapCountScope, httpapi.CapIssuesCountScope) } @@ -214,12 +217,12 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, httpapi.CapIssuesCloseAutoCloseMolecule, httpapi.CapIssuesListSearch, - httpapi.CapIssuesListNoAssignee, httpapi.CapIssuesListTitle, httpapi.CapListIDs, httpapi.CapIssuesUpdatePersistence, + httpapi.CapIssuesListNoAssignee, httpapi.CapIssuesListTitle, httpapi.CapListIDs, httpapi.CapIssuesUpdatePersistence, httpapi.CapDependenciesBlockingIsBlocked, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit, "+ "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule, issues.list.search, "+ - "issues.list.noAssignee, issues.list.title, issues.list.ids and issues.update.persistence tokens", behaviorCapabilities) + "issues.list.noAssignee, issues.list.title, issues.list.ids, issues.update.persistence and dependencies.blocking.isBlocked tokens", behaviorCapabilities) } } diff --git a/internal/storage/dolt/blocking_annotator_contract_test.go b/internal/storage/dolt/blocking_annotator_contract_test.go index d63631065..9e137e26f 100644 --- a/internal/storage/dolt/blocking_annotator_contract_test.go +++ b/internal/storage/dolt/blocking_annotator_contract_test.go @@ -71,6 +71,9 @@ func TestBlockingAnnotatorContract(t *testing.T) { t.Run("WritesNothing", func(t *testing.T) { conformance.RunBlockingAnnotatorWritesNothing(t, ctx, fixture) }) + t.Run("ReportsTheTransitiveBlockedFlag", func(t *testing.T) { + conformance.RunBlockingAnnotatorReportsTheTransitiveBlockedFlag(t, ctx, fixture) + }) } func newDoltBlockingAnnotatorFixture(t *testing.T, prefix string) (conformance.BlockingAnnotatorFixture, context.Context, func()) { diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index dc4332bb1..18056084b 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -608,6 +608,17 @@ func (r *dependencySQLRepositoryImpl) GetBlockingInfo(ctx context.Context, issue return info, nil } +// IsBlockedBatch reads the stored transitive is_blocked flag for ids from both +// planes through the shared body (issueops.IsBlockedBatchInTx) every store +// uses, so the unit-of-work answer cannot drift from the store's. +func (r *dependencySQLRepositoryImpl) IsBlockedBatch(ctx context.Context, ids []string) (map[string]bool, error) { + out, err := issueops.IsBlockedBatchInTx(ctx, r.runner, ids) + if err != nil { + return nil, fmt.Errorf("db: DependencySQLRepository.IsBlockedBatch: %w", err) + } + return out, nil +} + func (r *dependencySQLRepositoryImpl) GetBlockingInfoAcrossIssuesAndWisps(ctx context.Context, issueIDs []string) (domain.BlockingInfo, error) { perm, err := r.GetBlockingInfo(ctx, issueIDs, domain.DepListOpts{UseWispsTable: false}) if err != nil { diff --git a/internal/storage/domain/dependency.go b/internal/storage/domain/dependency.go index 0110c1c37..d4208560e 100644 --- a/internal/storage/domain/dependency.go +++ b/internal/storage/domain/dependency.go @@ -151,6 +151,10 @@ type DependencySQLRepository interface { GetBlockingInfo(ctx context.Context, issueIDs []string, opts DepListOpts) (BlockingInfo, error) GetBlockingInfoAcrossIssuesAndWisps(ctx context.Context, issueIDs []string) (BlockingInfo, error) IsBlocked(ctx context.Context, issueID string, opts DepListOpts) (bool, []string, error) + // IsBlockedBatch reads the stored transitive is_blocked flag for ids across + // both planes (issues win a cross-table collision); an id naming no row is + // absent from the map. + IsBlockedBatch(ctx context.Context, ids []string) (map[string]bool, error) // DeleteAllForIDs takes actor for the same reason Delete above does: the // edges it drops are journaled as dep_remove rows, and a cascade removal @@ -203,6 +207,9 @@ type DependencyUseCase interface { CountsByIssueIDs(ctx context.Context, issueIDs []string) (map[string]*types.DependencyCounts, error) GetBlockingInfo(ctx context.Context, issueIDs []string) (BlockingInfo, error) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) + // IsBlockedBatch is the repository read of the same name: the stored + // transitive is_blocked flag per id, both planes, absent = no row. + IsBlockedBatch(ctx context.Context, ids []string) (map[string]bool, error) GetForIssueIDs(ctx context.Context, ids []string) (map[string][]*types.Dependency, error) DetectCycles(ctx context.Context) ([][]*types.Issue, error) // DetectCycleReport is the shape issueops.CycleDetector publishes; see the @@ -617,6 +624,17 @@ func (u *dependencyUseCaseImpl) GetBlockingInfo(ctx context.Context, issueIDs [] return out, nil } +func (u *dependencyUseCaseImpl) IsBlockedBatch(ctx context.Context, ids []string) (map[string]bool, error) { + if len(ids) == 0 { + return map[string]bool{}, nil + } + out, err := u.depRepo.IsBlockedBatch(ctx, ids) + if err != nil { + return nil, fmt.Errorf("IsBlockedBatch: %w", err) + } + return out, nil +} + func (u *dependencyUseCaseImpl) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) { return u.isBlocked(ctx, issueID, false) } diff --git a/internal/storage/embeddeddolt/blocking_annotator_contract_test.go b/internal/storage/embeddeddolt/blocking_annotator_contract_test.go index 604e44436..f5ae720e6 100644 --- a/internal/storage/embeddeddolt/blocking_annotator_contract_test.go +++ b/internal/storage/embeddeddolt/blocking_annotator_contract_test.go @@ -75,6 +75,9 @@ func TestBlockingAnnotatorContract(t *testing.T) { t.Run("WritesNothing", func(t *testing.T) { conformance.RunBlockingAnnotatorWritesNothing(t, ctx, fixture) }) + t.Run("ReportsTheTransitiveBlockedFlag", func(t *testing.T) { + conformance.RunBlockingAnnotatorReportsTheTransitiveBlockedFlag(t, ctx, fixture) + }) } func newEmbeddedBlockingAnnotatorFixture(t *testing.T, te *testEnv, prefix string) conformance.BlockingAnnotatorFixture { diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 5c24bb39f..32432b7dc 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -720,13 +720,22 @@ func (s *Store) IsBlocked(ctx context.Context, issueID string) (bool, []string, // IsBlockedBatch preserves the external blocker invariant for batch callers. // The embedded Dolt implementation promotes this method from the wrapped // store, so it must be declared explicitly here rather than relying on -// IsBlocked alone. Like IsBlocked, it refuses over a store that serves its -// roles. +// IsBlocked alone. Over a store that serves its roles the flag comes from the +// server's annotator (dependencies.blocking.isBlocked), and +// remoteExternalBlocked adds the invariant unless the server already applies +// the policy; only IsBlocked still refuses there. func (s *Store) IsBlockedBatch(ctx context.Context, issueIDs []string) (map[string]bool, error) { blocked, err := s.inner.IsBlockedBatch(ctx, issueIDs) if err != nil { return nil, err } + if s.rolesAreRemote() { + // Over a store that serves its roles the raw edge read below is + // refused; the flag came from the server's annotator, and the + // external invariant is added from the edges of these rows only — + // unless the server already applies the policy itself. + return s.remoteExternalBlocked(ctx, issueIDs, blocked) + } deps, err := s.inner.GetDependencyRecordsForIssues(ctx, issueIDs) if err != nil { return nil, err diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index c5eaf52c9..6d0b09a71 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "slices" "testing" "time" @@ -157,6 +158,27 @@ func (f *fakeStore) IsBlockedBatch(_ context.Context, ids []string) (map[string] return result, nil } +// BlockingAnnotator serves the decoration a local backend derives for these +// rows: no local blockers, and when asked, isBlocked as each row's stored +// flag. +func (f *fakeStore) BlockingAnnotator() (publicops.BlockingAnnotator, error) { + return localAnnotator{f}, nil +} + +type localAnnotator struct{ f *fakeStore } + +func (a localAnnotator) AnnotateBlocking(ctx context.Context, req publicops.BlockingRequest) (publicops.BlockingResult, error) { + items := make([]publicops.IssueBlocking, 0, len(req.IDs)) + for _, id := range req.IDs { + item := publicops.IssueBlocking{ID: id} + if issue, _ := a.f.GetIssue(ctx, id); issue != nil && req.IncludeIsBlocked { + item.IsBlocked = boolPtr(a.f.isBlocked) + } + items = append(items, item) + } + return publicops.BlockingResult{Items: items}, nil +} + func (f *fakeStore) CloseIssueChecked(ctx context.Context, id, _ string, _ storage.CloseIssueOptions) (storage.CloseIssueResult, error) { issue, err := f.GetIssue(ctx, id) if err != nil { @@ -1008,6 +1030,29 @@ func TestIsBlockedBatchIncludesUnsatisfiedExternalRef(t *testing.T) { } } +// TestBlockingAnnotatorFlagsUnsatisfiedExternalRef: the annotation's +// is_blocked flag keeps IsBlockedBatch's invariant, so a row the stored column +// calls unblocked reads blocked while an external ref still blocks it. +func TestBlockingAnnotatorFlagsUnsatisfiedExternalRef(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + ref := "external:remote:payments" + raw := &fakeStore{ready: []*types.Issue{a, b}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, ref, types.DepBlocks)}, + }} + annotator, err := testStore(raw, &fakeStore{}, true).BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + + result, err := annotator.AnnotateBlocking(t.Context(), publicops.BlockingRequest{IDs: []string{a.ID, b.ID}, IncludeIsBlocked: true}) + if err != nil { + t.Fatalf("AnnotateBlocking: %v", err) + } + if got, want := blockedFlags(result), map[string]string{a.ID: "true", b.ID: "false"}; !maps.Equal(got, want) { + t.Fatalf("AnnotateBlocking is_blocked = %v, want %v", got, want) + } +} + func TestCloseIssueCheckedRefusesUnsatisfiedExternalRef(t *testing.T) { a := issue("be-a") ref := "external:remote:payments" @@ -1119,3 +1164,69 @@ func assertExternalCloseBlockers(t *testing.T, err error, issueID, ref string) { t.Errorf("refusal = %q, want %q", err.Error(), wantText) } } + +// TestIsBlockedBatchHonorsShippedExternalRef (bd-yewju3 store half): as +// GetReadyWork does, the flag leaves a row unblocked when its external ref's +// project has shipped it. +func TestIsBlockedBatchHonorsShippedExternalRef(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }} + foreign := &fakeStore{labels: map[string][]*types.Issue{ + "provides:payments": {{ID: "remote-done", Status: types.StatusClosed}}, + }} + + got, err := testStore(raw, foreign, true).IsBlockedBatch(t.Context(), []string{a.ID}) + if err != nil { + t.Fatalf("IsBlockedBatch: %v", err) + } + if want := map[string]bool{a.ID: false}; !maps.Equal(got, want) { + t.Fatalf("IsBlockedBatch = %v, want %v", got, want) + } +} + +// TestBlockingAnnotatorLeavesShippedExternalRefUnblocked: the store arm's +// served flag reads IsBlocked, so a shipped ref leaves the row unblocked. +func TestBlockingAnnotatorLeavesShippedExternalRefUnblocked(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }} + foreign := &fakeStore{labels: map[string][]*types.Issue{ + "provides:payments": {{ID: "remote-done", Status: types.StatusClosed}}, + }} + annotator, err := testStore(raw, foreign, true).BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + + result, err := annotator.AnnotateBlocking(t.Context(), publicops.BlockingRequest{IDs: []string{a.ID}, IncludeIsBlocked: true}) + if err != nil { + t.Fatalf("AnnotateBlocking: %v", err) + } + if got, want := blockedFlags(result), map[string]string{a.ID: "false"}; !maps.Equal(got, want) { + t.Fatalf("AnnotateBlocking is_blocked = %v, want %v", got, want) + } +} + +// TestBlockingAnnotatorAddsNoFlagUnasked: a request that did not ask for +// is_blocked answers exactly as before, even for an externally blocked row. +func TestBlockingAnnotatorAddsNoFlagUnasked(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }} + annotator, err := testStore(raw, &fakeStore{}, true).BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + + result, err := annotator.AnnotateBlocking(t.Context(), publicops.BlockingRequest{IDs: []string{a.ID}}) + if err != nil { + t.Fatalf("AnnotateBlocking: %v", err) + } + if got, want := blockedFlags(result), map[string]string{a.ID: "absent"}; !maps.Equal(got, want) { + t.Fatalf("AnnotateBlocking is_blocked = %v, want %v", got, want) + } +} diff --git a/internal/storage/externaldeps/remote_roles.go b/internal/storage/externaldeps/remote_roles.go index c99a6402f..5f6c5c598 100644 --- a/internal/storage/externaldeps/remote_roles.go +++ b/internal/storage/externaldeps/remote_roles.go @@ -433,10 +433,35 @@ func (a *remoteBlockingAnnotator) AnnotateBlocking(ctx context.Context, req issu for _, ref := range blockers[result.Items[i].ID] { result.Items[i].BlockedBy = appendUnique(result.Items[i].BlockedBy, ref) } + if req.IncludeIsBlocked && len(blockers[result.Items[i].ID]) > 0 { + result.Items[i].IsBlocked = boolPtr(true) + } } return result, nil } +// remoteExternalBlocked adds the external-dependency invariant to a remote +// store's IsBlockedBatch answer: an id with an unsatisfied external blocker +// is blocked. A server that enforces the policy already folded it in. +func (s *Store) remoteExternalBlocked(ctx context.Context, ids []string, blocked map[string]bool) (map[string]bool, error) { + if enforced, err := s.serverEnforcesPolicy(ctx); err != nil || enforced { + if err != nil { + return nil, err + } + return blocked, nil + } + external, err := s.externalBlockersFor(ctx, ids) + if err != nil { + return nil, err + } + for id, refs := range external { + if len(refs) > 0 { + blocked[id] = true + } + } + return blocked, nil +} + // ── TreeWalker ────────────────────────────────────────────────────── type remoteTreeWalker struct { diff --git a/internal/storage/externaldeps/remote_roles_composition_test.go b/internal/storage/externaldeps/remote_roles_composition_test.go index 9069e323d..308124e9b 100644 --- a/internal/storage/externaldeps/remote_roles_composition_test.go +++ b/internal/storage/externaldeps/remote_roles_composition_test.go @@ -599,8 +599,9 @@ func TestRemoteReadyClaimerWalksAContendedFront(t *testing.T) { } // TestRemoteBlockingAnnotatorMergesExternalBlockers: the served annotation's -// own blockers stay, the external ones join them, and the edges behind them -// are read in chunks the wire accepts. +// own blockers stay, the external ones join them without an is_blocked flag +// the request did not ask for, and the edges behind them are read in chunks +// the wire accepts. func TestRemoteBlockingAnnotatorMergesExternalBlockers(t *testing.T) { issues := make([]*types.Issue, 0, 150) ids := make([]string, 0, 150) @@ -623,6 +624,9 @@ func TestRemoteBlockingAnnotatorMergesExternalBlockers(t *testing.T) { blockedBy := make(map[string][]string, len(result.Items)) for _, item := range result.Items { blockedBy[item.ID] = item.BlockedBy + if item.IsBlocked != nil { + t.Errorf("%s is_blocked = %v unasked, want no flag", item.ID, *item.IsBlocked) + } } if want := []string{"be-local", paymentsRef}; !slices.Equal(blockedBy["be-000"], want) { t.Errorf("be-000 blocked by %v, want %v", blockedBy["be-000"], want) diff --git a/internal/storage/externaldeps/role_accessors.go b/internal/storage/externaldeps/role_accessors.go index 065ac9ed5..fe4dd17e4 100644 --- a/internal/storage/externaldeps/role_accessors.go +++ b/internal/storage/externaldeps/role_accessors.go @@ -132,11 +132,18 @@ func (a *blockingAnnotator) AnnotateBlocking(ctx context.Context, req issueops.B } if blocked { result.Items[i].BlockedBy = blockers + // The flag follows IsBlockedBatch's external invariant: an + // unsatisfied external blocker blocks, whatever the column says. + if req.IncludeIsBlocked { + result.Items[i].IsBlocked = boolPtr(true) + } } } return result, nil } +func boolPtr(v bool) *bool { return &v } + // TreeWalker preserves synthetic external leaves for the normal down-tree // request. Reverse walks do not follow a source's dependencies, and the // combined/up variants retain the backend's existing traversal semantics. diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 596ca2cc1..4467e13d8 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -451,6 +451,41 @@ type dependencyUseCase struct { policy *Store } +// IsBlockedBatch keeps the external blocker invariant on the stored column, +// as the store decorator's IsBlockedBatch does: an id with an unsatisfied +// external blocker is blocked whatever the column says. The served is_blocked +// flag is this read on bd serve's unit-of-work arm, so it decides by the +// edges and resolver GetReadyWork excludes by, read through the same unit of +// work. Unlike Ready it resolves only the refs of the ids the column +// answered: a client asks in chunks, and no chunk may reopen every foreign +// project the workspace names. An id naming no row stays absent. +func (u *dependencyUseCase) IsBlockedBatch(ctx context.Context, ids []string) (map[string]bool, error) { + blocked, err := u.DependencyUseCase.IsBlockedBatch(ctx, ids) + if err != nil || len(blocked) == 0 { + return blocked, err + } + records, err := u.GetExternalBlockingDependencyRecords(ctx) + if err != nil { + return nil, fmt.Errorf("external dependencies: %w", err) + } + answered := make(map[string][]*types.Dependency, len(blocked)) + for id := range blocked { + if deps, ok := records[id]; ok { + answered[id] = deps + } + } + state, err := u.policy.blockingStateFromRecords(ctx, answered) + if err != nil { + return nil, fmt.Errorf("external dependencies: %w", err) + } + for id := range blocked { + if len(state.refsByIssue[id]) > 0 { + blocked[id] = true + } + } + return blocked, nil +} + func (u *dependencyUseCase) GetDependencyTree(ctx context.Context, rootID string, opts domain.DepTreeOpts) ([]*types.TreeNode, error) { tree, err := u.DependencyUseCase.GetDependencyTree(ctx, rootID, opts) if err != nil || opts.Direction == domain.DepDirectionIn || len(tree) == 0 { diff --git a/internal/storage/externaldeps/uow_decorator_test.go b/internal/storage/externaldeps/uow_decorator_test.go index 3b6c024fb..5f1392486 100644 --- a/internal/storage/externaldeps/uow_decorator_test.go +++ b/internal/storage/externaldeps/uow_decorator_test.go @@ -3,7 +3,9 @@ package externaldeps import ( "context" "errors" + "maps" "slices" + "strconv" "testing" "time" @@ -12,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) type fakeUOWProvider struct { @@ -30,6 +33,10 @@ type fakeUOW struct { func (u *fakeUOW) IssueUseCase() domain.IssueUseCase { return u.issues } func (u *fakeUOW) DependencyUseCase() domain.DependencyUseCase { return u.deps } +// Close lets a role run over the fake: RunTxRead closes each unit of work it +// opens. +func (u *fakeUOW) Close(context.Context) {} + type fakeIssueUseCase struct { domain.IssueUseCase ready []*types.Issue @@ -117,6 +124,23 @@ type fakeDependencyUseCase struct { external map[string][]*types.Dependency records map[string][]*types.Dependency wispDeps map[string][]*types.Dependency + // isBlocked is the stored is_blocked column, per row; an id it does not + // name has no row. + isBlocked map[string]bool +} + +func (u *fakeDependencyUseCase) GetBlockingInfo(context.Context, []string) (domain.BlockingInfo, error) { + return domain.BlockingInfo{}, nil +} + +func (u *fakeDependencyUseCase) IsBlockedBatch(_ context.Context, ids []string) (map[string]bool, error) { + result := make(map[string]bool, len(ids)) + for _, id := range ids { + if blocked, ok := u.isBlocked[id]; ok { + result[id] = blocked + } + } + return result, nil } func (u *fakeDependencyUseCase) GetExternalBlockingDependencyRecords(context.Context) (map[string][]*types.Dependency, error) { @@ -163,6 +187,149 @@ func TestWrapUOWProviderFiltersProxiedReadyWork(t *testing.T) { } } +// TestWrapUOWProviderBlockedFlagAgreesWithReadyWork: the stored is_blocked +// column has no external term, so a row whose only blocker is an unsatisfied +// external ref reads false there. bd serve answers the served flag on this +// arm, so the flag must call that row blocked, as GetReadyWork does, without +// naming a row the column did not answer: neither an orphan edge's source nor +// an unasked row. +func TestWrapUOWProviderBlockedFlagAgreesWithReadyWork(t *testing.T) { + blocked, free, elsewhere := issue("be-blocked"), issue("be-free"), issue("be-elsewhere") + const ghost = "be-ghost" + inner := &fakeUOW{ + issues: &fakeIssueUseCase{ready: []*types.Issue{blocked, free, elsewhere}}, + deps: &fakeDependencyUseCase{ + external: map[string][]*types.Dependency{ + blocked.ID: {externalDep(blocked.ID, "external:remote:payments", types.DepBlocks)}, + elsewhere.ID: {externalDep(elsewhere.ID, "external:remote:payments", types.DepBlocks)}, + ghost: {externalDep(ghost, "external:remote:payments", types.DepBlocks)}, + }, + isBlocked: map[string]bool{blocked.ID: false, free.ID: false, elsewhere.ID: false}, + }, + } + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, func(ProjectName) (string, bool) { + return "", false + }, nil) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + ready, err := uw.IssueUseCase().GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(ready.Items); !slices.Equal(ids, []string{free.ID}) { + t.Fatalf("proxied ready IDs = %v, want [%s]", ids, free.ID) + } + + asked := []string{blocked.ID, free.ID, ghost} + flags, err := uw.DependencyUseCase().IsBlockedBatch(t.Context(), asked) + if err != nil { + t.Fatalf("IsBlockedBatch: %v", err) + } + if want := map[string]bool{blocked.ID: true, free.ID: false}; !maps.Equal(flags, want) { + t.Errorf("IsBlockedBatch = %v, want %v", flags, want) + } + + annotator, err := provider.(uow.BlockingAnnotatorSource).BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + result, err := annotator.AnnotateBlocking(t.Context(), publicops.BlockingRequest{IDs: asked, IncludeIsBlocked: true}) + if err != nil { + t.Fatalf("AnnotateBlocking: %v", err) + } + if got, want := blockedFlags(result), map[string]string{blocked.ID: "true", free.ID: "false", ghost: "absent"}; !maps.Equal(got, want) { + t.Errorf("AnnotateBlocking is_blocked = %v, want %v", got, want) + } +} + +// TestWrapUOWProviderBlockedFlagResolvesOnlyTheAskedRefs: a client asks for +// the served flag in chunks, so unlike Ready, which must resolve every +// external ref in the workspace, IsBlockedBatch opens only the foreign +// projects its asked rows name, neither an unasked row's nor an orphan +// edge's. +func TestWrapUOWProviderBlockedFlagResolvesOnlyTheAskedRefs(t *testing.T) { + blocked, free, elsewhere := issue("be-blocked"), issue("be-free"), issue("be-elsewhere") + // shipped's external blocker is SATISFIED: the foreign project's labelled + // index carries a closed provider of the capability. Its record is answered + // and resolved, and it must still read unblocked — the half of the + // predicate a "raise every answered external record" body would break. + shipped := issue("be-shipped") + const ghost = "be-ghost" + inner := &fakeUOW{deps: &fakeDependencyUseCase{ + external: map[string][]*types.Dependency{ + blocked.ID: {externalDep(blocked.ID, "external:remote:invoices", types.DepBlocks)}, + shipped.ID: {externalDep(shipped.ID, "external:remote:payments", types.DepBlocks)}, + elsewhere.ID: {externalDep(elsewhere.ID, "external:other:billing", types.DepBlocks)}, + ghost: {externalDep(ghost, "external:orphan:audit", types.DepBlocks)}, + }, + isBlocked: map[string]bool{blocked.ID: false, free.ID: false, elsewhere.ID: false, shipped.ID: false}, + }} + opens := make(map[string]int) + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, + func(project ProjectName) (string, bool) { return "/projects/" + string(project), true }, + func(_ context.Context, path string) (storage.DoltStorage, error) { + opens[path]++ + return &fakeStore{labels: map[string][]*types.Issue{ + "provides:payments": {{ID: "remote-done", Status: types.StatusClosed}}, + }}, nil + }, + ) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + + for _, tc := range []struct { + asked []string + want map[string]bool + wantOpens map[string]int + }{ + {asked: []string{free.ID, ghost}, want: map[string]bool{free.ID: false}, wantOpens: map[string]int{}}, + { + asked: []string{blocked.ID, free.ID}, + want: map[string]bool{blocked.ID: true, free.ID: false}, + wantOpens: map[string]int{"/projects/remote": 1}, + }, + { + asked: []string{shipped.ID, free.ID}, + want: map[string]bool{shipped.ID: false, free.ID: false}, + wantOpens: map[string]int{"/projects/remote": 1}, + }, + { + asked: []string{blocked.ID, shipped.ID}, + want: map[string]bool{blocked.ID: true, shipped.ID: false}, + wantOpens: map[string]int{"/projects/remote": 1}, + }, + } { + clear(opens) + flags, err := uw.DependencyUseCase().IsBlockedBatch(t.Context(), tc.asked) + if err != nil { + t.Fatalf("IsBlockedBatch(%v): %v", tc.asked, err) + } + if !maps.Equal(flags, tc.want) { + t.Errorf("IsBlockedBatch(%v) = %v, want %v", tc.asked, flags, tc.want) + } + if !maps.Equal(opens, tc.wantOpens) { + t.Errorf("IsBlockedBatch(%v) opened %v, want %v", tc.asked, opens, tc.wantOpens) + } + } +} + +// blockedFlags renders each annotated row's is_blocked flag, "absent" where +// the row carries none. +func blockedFlags(result publicops.BlockingResult) map[string]string { + flags := make(map[string]string, len(result.Items)) + for _, item := range result.Items { + flags[item.ID] = "absent" + if item.IsBlocked != nil { + flags[item.ID] = strconv.FormatBool(*item.IsBlocked) + } + } + return flags +} + func TestWrapUOWProviderKeepsUnitOfWorkUnwrappable(t *testing.T) { inner := &fakeUOW{} provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, nil, nil) diff --git a/internal/storage/issueops/blocking_annotation.go b/internal/storage/issueops/blocking_annotation.go index 49ef3a99c..bec8b48c8 100644 --- a/internal/storage/issueops/blocking_annotation.go +++ b/internal/storage/issueops/blocking_annotation.go @@ -85,5 +85,28 @@ func ExecuteBlockingAnnotation(ctx context.Context, tx DBTX, request publicops.B if err != nil { return publicops.BlockingResult{}, err } - return FinishBlockingAnnotation(anchors, blockedBy, blocks, parent), nil + result := FinishBlockingAnnotation(anchors, blockedBy, blocks, parent) + if request.IncludeIsBlocked { + flags, err := IsBlockedBatchInTx(ctx, tx, anchors) + if err != nil { + return publicops.BlockingResult{}, err + } + ApplyBlockedFlags(&result, flags) + } + return result, nil +} + +// ApplyBlockedFlags stamps BlockingRequest.IncludeIsBlocked's answer onto a +// finished annotation: each item whose id has a row in flags gets that row's +// stored transitive is_blocked flag, and an item with none keeps a nil flag. +// It is shared by every body that answers the role, so "absent from flags" +// means "no row" on all of them. +func ApplyBlockedFlags(result *publicops.BlockingResult, flags map[string]bool) { + for i := range result.Items { + blocked, ok := flags[result.Items[i].ID] + if !ok { + continue + } + result.Items[i].IsBlocked = &blocked + } } diff --git a/internal/storage/uow/blocking_annotator_contract_test.go b/internal/storage/uow/blocking_annotator_contract_test.go index a9df7af0c..dcf7bbafe 100644 --- a/internal/storage/uow/blocking_annotator_contract_test.go +++ b/internal/storage/uow/blocking_annotator_contract_test.go @@ -72,6 +72,9 @@ func TestBlockingAnnotatorContract(t *testing.T) { t.Run("WritesNothing", func(t *testing.T) { conformance.RunBlockingAnnotatorWritesNothing(t, ctx, fixture) }) + t.Run("ReportsTheTransitiveBlockedFlag", func(t *testing.T) { + conformance.RunBlockingAnnotatorReportsTheTransitiveBlockedFlag(t, ctx, fixture) + }) } func newUOWBlockingAnnotatorFixture(t *testing.T, ctx context.Context, prefix string) conformance.BlockingAnnotatorFixture { diff --git a/internal/storage/uow/blockingannotator.go b/internal/storage/uow/blockingannotator.go index 6f587a5e4..25f575980 100644 --- a/internal/storage/uow/blockingannotator.go +++ b/internal/storage/uow/blockingannotator.go @@ -59,6 +59,14 @@ func (a *blockingAnnotator) AnnotateBlocking(ctx context.Context, request public if err != nil { return publicops.BlockingResult{}, err } - return storageissueops.FinishBlockingAnnotation(anchors, info.BlockedBy, info.Blocks, info.Parent), nil + result := storageissueops.FinishBlockingAnnotation(anchors, info.BlockedBy, info.Blocks, info.Parent) + if request.IncludeIsBlocked { + flags, err := uw.DependencyUseCase().IsBlockedBatch(ctx, anchors) + if err != nil { + return publicops.BlockingResult{}, err + } + storageissueops.ApplyBlockedFlags(&result, flags) + } + return result, nil }) } diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 8626bf9d9..f1a9a31fd 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -203,6 +203,7 @@ func TestRecordingDependencyUseCaseCoversItsSurface(t *testing.T) { "GetIssueDependencyRecords": reads, "GetWispDependencyRecords": reads, "IsBlocked": reads, + "IsBlockedBatch": reads, "IsWispBlocked": reads, "IterWispWithIssueMetadata": reads, "IterWithIssueMetadata": reads, diff --git a/issueops/blockingannotator.go b/issueops/blockingannotator.go index e34645ee1..e9f6e58b7 100644 --- a/issueops/blockingannotator.go +++ b/issueops/blockingannotator.go @@ -28,6 +28,18 @@ type BlockingRequest struct { // for EdgeReadRequest: the answer is keyed by id and a second entry for the // same key carries no second fact. IDs []string + + // IncludeIsBlocked asks for each answered id's stored, TRANSITIVE + // is_blocked flag (IssueBlocking.IsBlocked) alongside the decoration. It + // is opt-in because the flag is a separate read and a listing that only + // prints the decoration has no use for it. + // + // It is the batched readiness projection a cache needs to answer Ready + // with the store's own predicate: the column GetReadyWork filters on, + // which propagates DOWN parent-child edges and reads waits-for and + // conditional-blocks gates, so BlockedBy (two edge types, direct only) + // cannot stand in for it. BlockedFlags is the one entry that asks for it. + IncludeIsBlocked bool } // IssueBlocking is one issue's blocking decoration. @@ -62,6 +74,13 @@ type IssueBlocking struct { // caller that needs every structural edge asks EdgeReader, which answers // with the rows. Parent string `json:"parent,omitempty"` + // IsBlocked is the issue's stored, transitive is_blocked flag — the same + // value storage's IsBlockedBatch answers and GetReadyWork filters on. It + // is set only when the request asked (BlockingRequest.IncludeIsBlocked) + // and the id names a row in either plane; it is nil otherwise, and nil + // never means "not blocked". An unsatisfied external blocker sets it, + // exactly as it does for storage's IsBlockedBatch. + IsBlocked *bool `json:"is_blocked,omitempty"` } // BlockingResult is the per-id answer. @@ -158,3 +177,55 @@ type BlockingAnnotator interface { // there is no Missing flag on it. AnnotateBlocking(ctx context.Context, req BlockingRequest) (BlockingResult, error) } + +// BlockedFlagsChunk is how many ids BlockedFlags names per AnnotateBlocking +// call. It equals the served listBlockingAnnotations bound (100 issue_id +// values; httpapi's maxDependencyAnchors, held equal by a test there), so a +// call through any transport stays inside a question the server accepts. +const BlockedFlagsChunk = 100 + +// BlockedFlags answers the batched transitive is_blocked read (storage's +// IsBlockedBatch: id -> flag, an id naming no row absent) through the +// BlockingAnnotator role with IncludeIsBlocked, BlockedFlagsChunk ids per +// call. It is the one library entry for that read on a role-served backend: +// the http client's IsBlockedBatch is this function over its annotator. +// +// Repeats and blank entries are dropped before asking (a blank id names no +// row, so it would be absent either way). An annotator that answers an item +// without a flag for an id it reports is treated as "no row", matching the +// absent-means-not-found contract; a failure of any chunk fails the call. +func BlockedFlags(ctx context.Context, annotator BlockingAnnotator, ids []string) (map[string]bool, error) { + out := make(map[string]bool, len(ids)) + distinct := make([]string, 0, len(ids)) + seen := make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id == "" { + continue + } + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + distinct = append(distinct, id) + } + for start := 0; start < len(distinct); start += BlockedFlagsChunk { + end := min(start+BlockedFlagsChunk, len(distinct)) + result, err := annotator.AnnotateBlocking(ctx, BlockingRequest{ + IDs: distinct[start:end], + IncludeIsBlocked: true, + }) + if err != nil { + return nil, err + } + for _, item := range result.Items { + if item.IsBlocked == nil { + continue + } + if _, asked := seen[item.ID]; !asked { + continue + } + out[item.ID] = *item.IsBlocked + } + } + return out, nil +} From ea0f6e1a46accfb7f7aeec63803278d507fa14d4 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 14 Oct 2026 11:47:45 +0000 Subject: [PATCH 08/76] feat(http): advertise issues.update.claim and route claims by it (S24a/b) (#7555) * feat(http): advertise issues.update.claim and route claims by it (S24a/b) updateIssue has served `claim` (upstream #6890), resolved on both planes so a wisp is claimable, with no capability token: a client could only find it by eating the skew 400, and gc's wire_compat preflight (which lists issues.update.claim as an optional token) never saw it, so gc claimed every wisp through claimIssue, whose Claimer role excludes the wisp plane. Server: CapIssuesUpdateClaim ("issues.update.claim") joins behaviorCapabilities. The spec names it on UpdateIssueRequest.claim and in the context capabilities prose (apigen regenerated, descriptions only). No request or response member changes; wire_revision stays 2. Client: Lifecycle.Update reads the token from the cached handshake before the dial. With it, every claim is one updateIssue call and its answer is final (no fallback on any error). Without it, a claim alone goes straight to claimIssue (no doomed 400 first; a wisp still refuses by name, W-ClaimRequest.Wisp, whose Why now names the token), and a claim beside a patch, a guard or a force override is refused before the dial with ErrUnsupported{Capability: issues.update.claim}. serverPredatesUpdateClaim (the error-sniffing fallback) is gone. Tests: the four capability lists (routes.go, wire/handshake.go, server_test, wire vocabulary_test) move together; unit tests for both routes (pre-dial refusal dials nothing; with the token nothing falls back); served tests claim a wisp through updateIssue with the token (zero claimIssue dials, post-claim revision stitched), refuse it by name without it (zero updateIssue claims, one claimIssue), and refuse a claim beside a patch without the token while the same request claims and patches the wisp with it. Live-mutated: ignoring the token fails the unit test. Co-Authored-By: Claude Opus 5.5 * test(http): count only ready --claim's requests in the claim-next check TestE2E_AdvertisedExternalPolicyIsPassedThrough counted every request since counter.reset(), so its "no by-id claim" check also saw the earlier `update --claim` step. The MC handshake mask does not advertise issues.update.claim, so this PR routes that claim to claimIssue (POST /v0/beads/issues/:claim) by design, and the check failed in Bazel / embedded Dolt tier (//backend/http:http_served_test shard 6 of 7), turning CI Gate / Required red. Count from the snapshot length taken just before `ready --claim`, so the check covers only the requests that command issues. The counter is not reset: the close-policy check below scans the same history. Co-Authored-By: Claude Opus 5.5 * fix(http): refuse a token-less template claim on the client's pre-read Against a server without issues.update.claim, Lifecycle.Update sends a claim alone to claimIssue (claimWithoutUpdateClaim) and returns before applyTemplateGuardForServer runs. claimIssue's role has no template check (bd-jkp9v3), so `bd update