diff --git a/CHANGELOG.md b/CHANGELOG.md index 986e0cb64..3eea684e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 multi-tenant embedder serving many workspaces needs, where a process-global credential cannot stand in for one tenant's own. EXPERIMENTAL, pin an exact beads version. +- Unlimited ready and query reads over the HTTP backend (`bd ready --limit + 0`, `bd query --limit 0`, and every library read asking for no limit) + return every row against any server, bounded on the client by the new + `BEADS_HTTP_UNLIMITED_READ_CAP` (a positive integer up to 1000000; default + 10000). The client never puts `limit=0` on the wire, which a `bd serve + --allow-non-loopback` refuses and which an older server answered with its + default page, silently capping the result: an unlimited ready or query + read is one request bounded at the cap plus one (one consistent snapshot). + Past the cap, or when a server reports more rows below the requested + bound, the read is refused with a typed error + (`httpclient.ErrUnlimitedReadCap`) naming the cap, never truncated. An + unlimited `bd list` is unchanged: it walks its keyset cursor a page at a + time and is bounded only by `--max-rows` / `BEADS_MAX_ROWS`. Unlike a + malformed `BEADS_MAX_ROWS`, which is ignored with a warning, a malformed + cap value fails every unlimited ready or query read rather than falling + back to the default. ### Changed diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index ded7e9e30..99a9a9c8b 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -179,7 +179,6 @@ shapes: source: gc bdstore.go bdReadyArgs (BdStore.Ready, ready --json --limit 0) args: [ready, --json, --limit, "0"] compare: ids - pending: {slice: "S15", reason: "ready --limit 0 omits limit and the server default page silently caps the answer (B3 #2)"} - id: gc-ready-ephemeral-assignee-unlimited source: gc bdstore.go bdReadyArgs with --include-ephemeral --assignee args: [ready, --json, --include-ephemeral, --assignee, worker, --limit, "0"] @@ -192,7 +191,6 @@ shapes: source: gc workquery.go bdReadyPoolDemandShell (work_query, --include-ephemeral --limit 0) args: [ready, --include-ephemeral, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit, "0"] compare: ids - pending: {slice: "S15", reason: "ready --limit 0 silently capped by the server's default page (B3 #2)"} - id: gc-ready-workflow-migration source: gc workquery.go bdReadyPoolDemandMigrationShell (--sort oldest --limit 0) args: [ready, --include-ephemeral, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit, "0"] @@ -286,7 +284,7 @@ shapes: - id: mc-list-id source: mc skills/pr-review (list --all --id --json --limit) args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] - pending: {slice: "S6c+S15+S15b", reason: "list stub (B3 #1); --limit 0 (B3 #3); --id refused by the list encoder (B3 #11)"} + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --id refused by the list encoder (B3 #11)"} - id: mc-list-assignee-status source: mc review summaries (list --assignee --json --limit --status) args: [list, --assignee, alice, --json, --limit, "10", --status, open] @@ -316,7 +314,6 @@ shapes: source: gc workquery.go bdQueryEphemeralStatusShell (--limit=0, wisp tier) args: [query, --json, "ephemeral=true AND status=open", --limit=0] compare: ids - pending: {slice: "S15", reason: "query --limit=0 sends limit=0 and a non-loopback server refuses unlimited reads (B3 #3)"} - id: mc-query source: mc pr_merge.py (bd query) args: [query, --json, "status=open AND priority<=1", --limit, "50"] diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 4b0f74f0e..ea7f30ec2 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -51,6 +51,7 @@ go_library( "sweeper.go", "target.go", "target_seam.go", + "unlimited_read.go", "unsupported.go", "unsupported_gen.go", "vocabulary.go", @@ -156,10 +157,12 @@ go_test( "served_relations_test.go", "served_releaser_test.go", "served_test.go", + "served_unlimited_read_test.go", "served_wrong_server_test.go", "store_dependent_test.go", "store_test.go", "target_test.go", + "unlimited_read_test.go", "watch_test.go", "wave2b_roles_test.go", "wave2c_claimnext_test.go", diff --git a/internal/httpclient/bridge.go b/internal/httpclient/bridge.go index f6a1baccd..32f804d9d 100644 --- a/internal/httpclient/bridge.go +++ b/internal/httpclient/bridge.go @@ -117,6 +117,17 @@ func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.Wor if err != nil { return nil, false, s.inexpressible(op, err) } + // A legacy filter spells unlimited as Limit <= 0, and the encoder therefore + // emits no `limit` for it — which the server reads as ITS default of 100, + // not as unlimited. That was a silent cap on every `bd ready --limit 0` + // (and every gc routed-pool queue read through it). The pager decides the + // bound instead: the client cap plus one, refused past the cap rather than + // truncated, and never the wire's own `limit=0` (unlimited_read.go). + readCap, err := boundUnlimitedPage(params, filter.Limit <= 0) + if err != nil { + return nil, false, err + } + readCap = tightenUnlimitedPage(params, readCap, filter.MaxRows) var body apigen.ReadyPage if err := s.dispatch(ctx, wire.Request{ Op: wire.OpListReadyWork, @@ -141,6 +152,9 @@ func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.Wor Source: filter.MaxRowsSource, } } + if err := checkUnlimitedPage(wire.OpListReadyWork, readCap, len(rows), body.HasMore); err != nil { + return nil, false, err + } return rows, body.HasMore, nil } diff --git a/internal/httpclient/dispatch.go b/internal/httpclient/dispatch.go index 8322da940..a168668d8 100644 --- a/internal/httpclient/dispatch.go +++ b/internal/httpclient/dispatch.go @@ -48,6 +48,11 @@ func (s *Store) dispatch(ctx context.Context, req wire.Request, out any) error { if s.wire == nil { return fmt.Errorf("%w: cannot dial %s", ErrNoTransport, s.target) } + // Before the preflight, so the tripwire costs no handshake: `limit=0` is + // refused off loopback, and this client never sends it (unlimited_read.go). + if err := refuseUnlimitedOnWire(req); err != nil { + return err + } if err := s.wire.Preflight(ctx, req.Op); err != nil { return err } diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 43d61f55d..41be7e31e 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -720,6 +720,9 @@ func (b *builder) intPtr(param string, value *int) { // intPositive emits a plain int only when it bounds something. The legacy // filters spell "no bound" as 0, so a zero here is an absent parameter rather // than the wire's `limit=0`, which means unlimited and is refused off loopback. +// An absent `limit` is the SERVER's default, not unlimited, so a caller that +// means unlimited must not stop here: the ready bridge's pager writes the +// client-capped bound itself (httpclient/unlimited_read.go). func (b *builder) intPositive(param string, value int) { if value > 0 { b.v.Set(param, strconv.Itoa(value)) diff --git a/internal/httpclient/role_querier.go b/internal/httpclient/role_querier.go index 0129a99f1..3d61af937 100644 --- a/internal/httpclient/role_querier.go +++ b/internal/httpclient/role_querier.go @@ -35,6 +35,12 @@ func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issu if err != nil { return issueops.IssuePage{}, q.store.inexpressible("Querier.Query", err) } + // An explicit Limit of 0 is bounded at the client cap rather than sent as + // `limit=0`, which a non-loopback server refuses (unlimited_read.go). + readCap, err := boundUnlimitedPage(params, req.Limit != nil && *req.Limit == 0) + if err != nil { + return issueops.IssuePage{}, err + } var body apigen.QueryPage if err := q.store.dispatch(ctx, wire.Request{ Op: wire.OpQueryIssues, @@ -44,6 +50,9 @@ func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issu }, &body); err != nil { return issueops.IssuePage{}, err } + if err := checkUnlimitedPage(wire.OpQueryIssues, readCap, len(body.Items), body.HasMore); err != nil { + return issueops.IssuePage{}, err + } // NEVER projected: issueops.QueryRequest has no Brief member, so this role // cannot ask for one and its rows are always whole. The `false` is the // compiler asking the question rather than a value to tune — if the query diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index 8d3333ab5..d1758a083 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -32,11 +32,19 @@ type httpReader struct{ store *Store } // server's own has_more ARE the page contract. An Offset is refused by the // encoder rather than dropped — ready work carries no keyset position, so there // is no honest way to page it (E-ReadyRequest.Offset). +// +// An explicit Limit of 0 (unlimited) is NOT sent as `limit=0`: it becomes one +// request bounded at the client cap plus one, refused past the cap rather than +// truncated (unlimited_read.go). func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { params, err := encode.ReadyParams(req) if err != nil { return issueops.IssuePage{}, r.store.inexpressible("Reader.Ready", err) } + readCap, err := boundUnlimitedPage(params, req.Limit != nil && *req.Limit == 0) + if err != nil { + return issueops.IssuePage{}, err + } var body apigen.ReadyPage if err := r.store.dispatch(ctx, wire.Request{ Op: wire.OpListReadyWork, @@ -46,6 +54,9 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue }, &body); err != nil { return issueops.IssuePage{}, err } + if err := checkUnlimitedPage(wire.OpListReadyWork, readCap, len(body.Items), body.HasMore); err != nil { + return issueops.IssuePage{}, err + } return issueops.IssuePage{Items: wireRows(body.Items, req.Brief), HasMore: body.HasMore}, nil } diff --git a/internal/httpclient/served_external_dependency_policy_test.go b/internal/httpclient/served_external_dependency_policy_test.go index e530a3085..3b11a08bd 100644 --- a/internal/httpclient/served_external_dependency_policy_test.go +++ b/internal/httpclient/served_external_dependency_policy_test.go @@ -124,12 +124,15 @@ func TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability // TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage pins, // against a real served server, the shape the decorator once misread: an -// unlimited ready listing over http comes back as the server's default page -// (the wire omits a zero limit) while its total counts the whole ready set. -// Text-mode `bd ready` counts through CountReadyWork, which always fetches -// unlimited, so an externally blocked issue sorted past that page stayed in -// its count when the decorator took the zero limit for a window holding every -// row. +// unlimited ready listing over http used to come back as the server's default +// page (the wire omitted a zero limit) while its total counted the whole ready +// set. Text-mode `bd ready` counted through CountReadyWork's unlimited fetch, +// so an externally blocked issue sorted past that page stayed in its count when +// the decorator took the zero limit for a window holding every row. An +// unlimited ready read now returns every row (one request bounded at the client +// cap, unlimited_read.go), and CountReadyWork asks for one row and lets the +// server count the rest; the count and the default page must still leave that +// issue out. func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *testing.T) { env := newServedEnv(t, "hixp") ctx := t.Context() @@ -165,19 +168,20 @@ func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *t if err != nil { t.Fatalf("BuildReadyCountFilter: %v", err) } - // The premise this test pins: undecorated, the unlimited listing is the - // default page, the blocked issue is not on it, and the total counts it. + // The premise this test pins: undecorated, the unlimited listing holds + // every ready row (no longer only the server's default page), and the + // blocked issue sorts past where the default page ends. rows, total, err := env.subject.GetReadyWorkWithCountsAndTotal(ctx, countFilter) if err != nil { t.Fatalf("undecorated GetReadyWorkWithCountsAndTotal over http: %v", err) } - if len(rows) != workapi.DefaultReadyLimit || total != workapi.DefaultReadyLimit+2 { - t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want the %d-row default page, total %d; this test's premise no longer holds", - len(rows), total, workapi.DefaultReadyLimit, workapi.DefaultReadyLimit+2) + if len(rows) != workapi.DefaultReadyLimit+2 || total != workapi.DefaultReadyLimit+2 { + t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want every ready row, %d and total %d; this test's premise no longer holds", + len(rows), total, workapi.DefaultReadyLimit+2, workapi.DefaultReadyLimit+2) } - for _, row := range rows { - if row != nil && row.Issue != nil && row.ID == blockedID { - t.Fatalf("%q is on the default page; this test needs it past the page", blockedID) + for i, row := range rows { + if row != nil && row.Issue != nil && row.ID == blockedID && i < workapi.DefaultReadyLimit { + t.Fatalf("%q is row %d, inside the default page; this test needs it past the page", blockedID, i) } } @@ -443,7 +447,8 @@ func TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter(t *test // fetches a window wider than the limit to make room for the rows it drops, // and the http bridge caps the rows it fetched, so the cap counted rows the // caller was never handed. A SQL store caps the page it delivers, and so must -// this path. +// this path. An unlimited read keeps the cap on its request, grown by the rows +// the decorator may drop, and refuses in the caller's name past it. func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { env := newServedEnv(t, "hixm") ctx := t.Context() @@ -478,8 +483,12 @@ func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { {name: "cap equals the limit", limit: 2, maxRows: 2, want: free[:2]}, // The window holds all four rows, three of them kept. {name: "cap below the limit", limit: 5, maxRows: 3, want: free}, - // The server's default page, again all four. + // The request is sized by the cap grown by the blocked row: all four + // rows, three of them kept. {name: "unlimited", limit: 0, maxRows: 3, want: free}, + // The four rows outnumber the cap grown by the blocked row, so at + // least three would be kept, and the caller's cap of two fires. + {name: "unlimited, kept rows exceed the cap", limit: 0, maxRows: 2}, // The three rows kept exceed the caller's cap, which still fires. {name: "kept rows exceed the cap", limit: 5, maxRows: 2}, } { diff --git a/internal/httpclient/served_unlimited_read_test.go b/internal/httpclient/served_unlimited_read_test.go new file mode 100644 index 000000000..0e636251c --- /dev/null +++ b/internal/httpclient/served_unlimited_read_test.go @@ -0,0 +1,511 @@ +//go:build cgo + +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/url" + "slices" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage/externaldeps" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// Unlimited reads against a server in NON-LOOPBACK mode (S15). +// +// The in-process server binds 127.0.0.1 — tests never listen beyond loopback — +// but is started with AllowNonLoopback, which is exactly the switch the +// server's unlimited-read refusal reads (httpapi allowUnlimited). So every +// `limit=0` this client sent would meet the production 400, and every answer +// below that comes back whole is proof that none was sent. +// +// The fixture is unlimitedRows rows: more than the server's ready default +// (workapi.DefaultReadyLimit, 100), so a read that silently fell back to that +// default — `bd ready --limit 0` before S15 — comes back short and fails the +// count, and more than one walkPageSize, so the list walk crosses pages. + +const unlimitedRows = 150 + +// nonLoopbackMode serves the whole surface as a --allow-non-loopback server +// would. InsecureNoAuth is the posture rule's required waiver for a +// non-loopback server with no token file; the listener itself stays on +// 127.0.0.1. +func nonLoopbackMode(cfg *httpapi.Config) { + cfg.AllowNonLoopback = true + cfg.InsecureNoAuth = true +} + +// limitSpy records the `limit` of every issue-listing request that reaches the +// wire, after dispatch's own tripwire. +type limitSpy struct { + mu sync.Mutex + sent []string // "op limit=" +} + +func (s *limitSpy) record(req *wire.Request) { + if !unlimitedSpellingOps[req.Op] { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.sent = append(s.sent, fmt.Sprintf("%s limit=%s", req.Op, req.Query.Get("limit"))) +} + +func (s *limitSpy) reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.sent = nil +} + +func (s *limitSpy) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.sent...) +} + +// requestsFor is requests narrowed to one operation's. +func (s *limitSpy) requestsFor(op string) []string { + var got []string + for _, r := range s.requests() { + if strings.HasPrefix(r, op+" ") { + got = append(got, r) + } + } + return got +} + +// assertBoundedOnWire fails if any recorded request carried limit=0 or +// no limit at all (the server's default, which is the silent-truncation leg). +func (s *limitSpy) assertBoundedOnWire(t *testing.T) { + t.Helper() + got := s.requests() + if len(got) == 0 { + t.Fatal("no listing request was recorded; the spy is not on the path") + } + for _, r := range got { + if strings.HasSuffix(r, " limit=0") || strings.HasSuffix(r, " limit=") { + t.Errorf("request %q went out unbounded (all requests: %v)", r, got) + } + } +} + +type unlimitedEnv struct { + *servedEnv + spy *limitSpy + store *Store + scope string +} + +func newUnlimitedEnv(t *testing.T) *unlimitedEnv { + t.Helper() + e := newServedEnv(t, "unl", nonLoopbackMode) + ctx := t.Context() + scope := "unlimited-scope" + issues := make([]*types.Issue, 0, unlimitedRows) + for i := range unlimitedRows { + id := fmt.Sprintf("unl-%03d", i) + issues = append(issues, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: i % 4, + IssueType: types.TypeTask, Labels: []string{scope}, + }) + } + if err := e.reference.CreateIssues(ctx, issues, "seed"); err != nil { + t.Fatalf("seed %d rows: %v", unlimitedRows, err) + } + spy := &limitSpy{} + store := New(e.subject.target, tamperedWire{WireClient: e.subject.wire, tamper: spy.record}, nil) + return &unlimitedEnv{servedEnv: e, spy: spy, store: store, scope: scope} +} + +func (u *unlimitedEnv) reader(t *testing.T) issueops.Reader { + t.Helper() + r, err := u.store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return r +} + +func (u *unlimitedEnv) querier(t *testing.T) issueops.Querier { + t.Helper() + q, err := u.store.Querier() + if err != nil { + t.Fatalf("Querier(): %v", err) + } + return q +} + +// listAll is an unlimited list: a cursor walk that only maxRows bounds (0 for +// none). It returns the row count. +func (u *unlimitedEnv) listAll(t *testing.T, maxRows int) (int, error) { + t.Helper() + page, err := u.reader(t).List(t.Context(), issueops.ListRequest{ + Labels: []string{u.scope}, Limit: ptrTo(0), MaxRows: maxRows, + }) + if err == nil && page.HasMore { + t.Error("an unlimited list reported HasMore") + } + return len(page.Items), err +} + +// readFn is one of the unlimited reads under test, returning its row count. +type readFn func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) + +// cursorlessReads are the unlimited reads that are ONE request — ready and +// query publish no cursor — and so the ones the client cap bounds. +func cursorlessReads() []struct { + name string + read readFn +} { + zero := func() *int { z := 0; return &z } + return []struct { + name string + read readFn + }{ + {"ready via the Reader role", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + page, err := u.reader(t).Ready(ctx, issueops.ReadyRequest{Labels: []string{u.scope}, Limit: zero()}) + return len(page.Items), err + }}, + {"bd ready via the WorkFilter bridge", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: zero(), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, err := u.store.GetReadyWork(ctx, filter) + return len(rows), err + }}, + {"bd ready --json via the WorkFilter bridge with a total", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: zero(), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, total, err := u.store.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err == nil && total != len(rows) { + t.Errorf("total = %d for %d rows; an unlimited page is the whole set", total, len(rows)) + } + return len(rows), err + }}, + {"query", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + page, err := u.querier(t).Query(ctx, issueops.QueryRequest{Expression: "label=" + u.scope, Limit: zero()}) + return len(page.Items), err + }}, + } +} + +// TestServedUnlimitedReadsOffLoopback: ready, query and list at limit 0 return +// every row from a non-loopback-mode server, and never put `limit=0` (or a +// missing limit) on the wire. +// +// GOES RED when the bridge sends no limit for Limit 0 (the server's 100-row +// default: 100 != 150), when the role or the querier forwards `limit=0` (the +// server's 400, or the dispatch tripwire), or when the list walk stops short of +// the fixture. +func TestServedUnlimitedReadsOffLoopback(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + + restore := walkPageSize + walkPageSize = 40 + t.Cleanup(func() { walkPageSize = restore }) + + // PREMISE 1: the server really is in non-loopback mode — a raw `limit=0` + // that bypasses the client's dispatch is refused. + var raw apigen.ReadyPage + err := u.subject.wire.Do(ctx, wire.Request{ + Op: wire.OpListReadyWork, Method: "GET", Path: wire.PathReady, + Query: url.Values{"limit": {"0"}}, + }, &raw) + if err == nil || !strings.Contains(err.Error(), "loopback-only") { + t.Fatalf("a raw limit=0 was not refused by the server (err %v); the fixture is not in non-loopback mode", err) + } + + // PREMISE 2: the fixture exceeds the server's ready default, so a read + // that fell back to it cannot pass the count below. + page, err := u.reader(t).Ready(ctx, issueops.ReadyRequest{Labels: []string{u.scope}}) + if err != nil { + t.Fatalf("default-limit ready: %v", err) + } + if len(page.Items) != workapi.DefaultReadyLimit || !page.HasMore { + t.Fatalf("default-limit ready returned %d rows (has_more %v), want the server default %d with more", + len(page.Items), page.HasMore, workapi.DefaultReadyLimit) + } + + for _, tc := range cursorlessReads() { + t.Run(tc.name, func(t *testing.T) { + u.spy.reset() + n, err := tc.read(t, ctx, u) + if err != nil { + t.Fatalf("unlimited read: %v", err) + } + if n != unlimitedRows { + t.Errorf("unlimited read returned %d rows, want all %d", n, unlimitedRows) + } + u.spy.assertBoundedOnWire(t) + }) + } + + // The list walk returns every row, and crosses pages rather than asking + // for everything at once. + t.Run("list", func(t *testing.T) { + u.spy.reset() + n, err := u.listAll(t, 0) + if err != nil { + t.Fatalf("unlimited list: %v", err) + } + if n != unlimitedRows { + t.Errorf("unlimited list returned %d rows, want all %d", n, unlimitedRows) + } + u.spy.assertBoundedOnWire(t) + if got := len(u.spy.requests()); got < unlimitedRows/walkPageSize { + t.Errorf("the unlimited list took %d requests at %d rows a page; it did not walk", got, walkPageSize) + } + }) +} + +// TestServedUnlimitedReadCapRefuses: past BEADS_HTTP_UNLIMITED_READ_CAP every +// unlimited ready or query read is refused with *UnlimitedReadCapError, never +// truncated; at the cap exactly, the whole answer comes back. An unlimited list +// is a cursor walk, and the cap does not apply to it: only MaxRows bounds it. +// +// GOES RED when a cursorless path truncates to the cap instead of refusing, or +// ignores it; when a full cap+1 page that reports has_more is taken for a +// server truncation; or when the list walk is capped. +func TestServedUnlimitedReadCapRefuses(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + + for _, tc := range cursorlessReads() { + t.Run(tc.name, func(t *testing.T) { + // At unlimitedRows-1 the cap+1 page is the whole set; at + // unlimitedRows-2 it is a full page with has_more set, which is how + // a real workspace overflows the cap. Both are the cap's refusal. + for _, readCap := range []int{unlimitedRows - 1, unlimitedRows - 2} { + t.Setenv(UnlimitedReadCapEnv, fmt.Sprint(readCap)) + u.spy.reset() + n, err := tc.read(t, ctx, u) + var capErr *UnlimitedReadCapError + if !errors.As(err, &capErr) { + t.Fatalf("cap %d: got %d rows, err %v; want *UnlimitedReadCapError", readCap, n, err) + } + if !errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("cap %d: the refusal does not match ErrUnlimitedReadCap: %v", readCap, err) + } + if capErr.Cap != readCap || capErr.Found != readCap+1 || capErr.ServerTruncated { + t.Errorf("cap %d: refusal = %+v, want Cap %d Found %d and no server truncation", + readCap, capErr, readCap, readCap+1) + } + if !strings.Contains(err.Error(), UnlimitedReadCapEnv) { + t.Errorf("cap %d: the refusal does not name %s: %v", readCap, UnlimitedReadCapEnv, err) + } + u.spy.assertBoundedOnWire(t) + } + + t.Setenv(UnlimitedReadCapEnv, fmt.Sprint(unlimitedRows)) + n, err := tc.read(t, ctx, u) + if err != nil || n != unlimitedRows { + t.Fatalf("at the cap exactly: got %d rows, err %v; want all %d", n, err, unlimitedRows) + } + + t.Setenv(UnlimitedReadCapEnv, "lots") + if _, err := tc.read(t, ctx, u); err == nil || !strings.Contains(err.Error(), UnlimitedReadCapEnv) { + t.Errorf("a malformed cap was not refused: %v", err) + } + }) + } + + t.Run("list is not capped", func(t *testing.T) { + restore := walkPageSize + walkPageSize = 40 + t.Cleanup(func() { walkPageSize = restore }) + + for _, readCap := range []string{"10", "lots"} { + t.Setenv(UnlimitedReadCapEnv, readCap) + if n, err := u.listAll(t, 0); err != nil || n != unlimitedRows { + t.Errorf("cap %q: the unlimited list returned %d rows, err %v; want all %d", readCap, n, err, unlimitedRows) + } + } + + // MaxRows is the walk's only bound, and past it the refusal is the + // caller's own. + t.Setenv(UnlimitedReadCapEnv, "10") + _, err := u.listAll(t, 100) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) || tooMany.Cap != 100 || tooMany.Found != 101 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("the unlimited list past MaxRows 100: err %v; want *ErrTooManyRows Cap 100 Found 101", err) + } + if n, err := u.listAll(t, unlimitedRows); err != nil || n != unlimitedRows { + t.Errorf("MaxRows at the fixture: %d rows, err %v; want all %d", n, err, unlimitedRows) + } + }) +} + +// TestServedReadyBridgeMaxRowsSizesTheRequest: when an unlimited bridge read +// carries a MaxRows as well, the tighter of it and the cap sizes the request, +// so a small MaxRows never makes the server buffer up to the cap, and an answer +// past it is refused in the name of the bound that fired. `bd ready` reads +// through the external-dependency decorator, which drops an externally +// blocked row itself against this server; the same holds through it, with the +// request grown by the rows it may drop and the cap still in force. +// +// GOES RED when the bridge sends the cap's limit under a tighter MaxRows, or +// reports an answer past MaxRows as the read cap's refusal; or when the +// decorator takes MaxRows off an unlimited read, does not grow it by the +// blocked rows, reports the refusal against the grown bound, or lets the +// growth lift the request past the cap. +func TestServedReadyBridgeMaxRowsSizesTheRequest(t *testing.T) { + u := newUnlimitedEnv(t) + read := func(t *testing.T, ready func(context.Context, types.WorkFilter) ([]*types.Issue, error), maxRows int) (int, error) { + t.Helper() + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: ptrTo(0), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + filter.MaxRows = maxRows + u.spy.reset() + rows, err := ready(t.Context(), filter) + return len(rows), err + } + sent := func(limit int) []string { return []string{fmt.Sprintf("%s limit=%d", wire.OpListReadyWork, limit)} } + + t.Setenv(UnlimitedReadCapEnv, "10") + _, err := read(t, u.store.GetReadyWork, 5) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) || tooMany.Cap != 5 || tooMany.Found != 6 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("MaxRows 5 under cap 10: err %v; want *ErrTooManyRows Cap 5 Found 6", err) + } + if got := u.spy.requests(); !slices.Equal(got, sent(6)) { + t.Errorf("MaxRows 5 under cap 10 sent %v, want %v", got, sent(6)) + } + + _, err = read(t, u.store.GetReadyWork, 50) + var capErr *UnlimitedReadCapError + if !errors.As(err, &capErr) || capErr.Cap != 10 || capErr.Found != 11 || capErr.ServerTruncated { + t.Errorf("MaxRows 50 over cap 10: err %v; want *UnlimitedReadCapError Cap 10 Found 11", err) + } + if got := u.spy.requests(); !slices.Equal(got, sent(11)) { + t.Errorf("MaxRows 50 over cap 10 sent %v, want %v", got, sent(11)) + } + + // Both bounds clear the fixture: every row, in the request MaxRows sized. + t.Setenv(UnlimitedReadCapEnv, "1000") + if n, err := read(t, u.store.GetReadyWork, unlimitedRows+10); err != nil || n != unlimitedRows { + t.Errorf("MaxRows %d under cap 1000: %d rows, err %v; want all %d", unlimitedRows+10, n, err, unlimitedRows) + } + if got := u.spy.requests(); !slices.Equal(got, sent(unlimitedRows+11)) { + t.Errorf("MaxRows %d under cap 1000 sent %v, want %v", unlimitedRows+10, got, sent(unlimitedRows+11)) + } + + // Through the decorator, with unl-000 externally blocked: the request asks + // for one row more per blocked issue, and past that bound more than MaxRows + // rows would still be kept, so the refusal is still the caller's own. The + // decorator's blocking state is loaded with a list walk, which is not the + // read's. + if enforced, err := u.store.ServerEnforcesExternalDependencyPolicy(t.Context()); err != nil || enforced { + t.Fatalf("ServerEnforcesExternalDependencyPolicy = %v, %v; this test needs a server that leaves the policy to the client", enforced, err) + } + if err := u.addDependency(t.Context(), &types.Dependency{ + IssueID: "unl-000", DependsOnID: "external:otherproj:cap", Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(u.store, nil, nil).GetReadyWork + + t.Setenv(UnlimitedReadCapEnv, "10") + _, err = read(t, decorated, 5) + if !errors.As(err, &tooMany) || tooMany.Cap != 5 || tooMany.Found != 6 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("decorated, MaxRows 5 under cap 10: err %v; want *ErrTooManyRows Cap 5 Found 6", err) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(7)) { + t.Errorf("decorated, MaxRows 5 under cap 10 sent %v, want %v", got, sent(7)) + } + + // MaxRows and the blocked row together reach past the cap, which sizes + // the request and refuses, as it does for the bridge alone. + _, err = read(t, decorated, 50) + if !errors.As(err, &capErr) || capErr.Cap != 10 || capErr.Found != 11 || capErr.ServerTruncated { + t.Errorf("decorated, MaxRows 50 over cap 10: err %v; want *UnlimitedReadCapError Cap 10 Found 11", err) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(11)) { + t.Errorf("decorated, MaxRows 50 over cap 10 sent %v, want %v", got, sent(11)) + } + + t.Setenv(UnlimitedReadCapEnv, "1000") + if n, err := read(t, decorated, unlimitedRows+10); err != nil || n != unlimitedRows-1 { + t.Errorf("decorated, MaxRows %d under cap 1000: %d rows, err %v; want all %d but unl-000", unlimitedRows+10, n, err, unlimitedRows-1) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(unlimitedRows+12)) { + t.Errorf("decorated, MaxRows %d under cap 1000 sent %v, want %v", unlimitedRows+10, got, sent(unlimitedRows+12)) + } +} + +// TestServedReadyCountFallbackStaysBounded: text-mode `bd ready` counts through +// the external-dependency decorator, and against a server that does not apply +// that policy itself (every released server, and this harness) the decorator +// counts over a store with no indexed counter. It asks for one row and lets +// the server count the rest, so a ready set larger than +// BEADS_HTTP_UNLIMITED_READ_CAP is counted rather than refused, and never +// downloaded to report one number. +// +// GOES RED when the count's fallback asks for the whole ready set again: past +// the cap that read is refused, and the spy sees more than the one row. +func TestServedReadyCountFallbackStaysBounded(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + t.Setenv(UnlimitedReadCapEnv, "10") + + if enforced, err := u.store.ServerEnforcesExternalDependencyPolicy(ctx); err != nil || enforced { + t.Fatalf("ServerEnforcesExternalDependencyPolicy = %v, %v; this test needs a server that leaves the policy to the client", enforced, err) + } + counter, err := externaldeps.New(u.store, nil, nil).ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + // count returns the total and the count's ready listings; the decorator's + // blocking state is loaded with a list walk, which is not the count's. + count := func(t *testing.T) (int64, []string) { + t.Helper() + u.spy.reset() + got, err := counter.CountReady(ctx, issueops.ReadyRequest{Labels: []string{u.scope}}) + if err != nil { + t.Fatalf("CountReady over %d ready rows at cap 10: %v", unlimitedRows, err) + } + return got.Total, u.spy.requestsFor(wire.OpListReadyWork) + } + + total, ready := count(t) + if total != unlimitedRows { + t.Errorf("count = %d, want %d", total, unlimitedRows) + } + if want := []string{wire.OpListReadyWork + " limit=1"}; !slices.Equal(ready, want) { + t.Errorf("the count listed %v, want %v: one row, the server counting the rest", ready, want) + } + + // An externally blocked issue makes the decorator size the filtered set + // itself, at an explicit limit the cap does not apply to. + if err := u.addDependency(ctx, &types.Dependency{ + IssueID: "unl-000", DependsOnID: "external:otherproj:cap", Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + if total, _ := count(t); total != unlimitedRows-1 { + t.Errorf("count with unl-000 externally blocked = %d, want %d", total, unlimitedRows-1) + } + u.spy.assertBoundedOnWire(t) +} diff --git a/internal/httpclient/unlimited_read.go b/internal/httpclient/unlimited_read.go new file mode 100644 index 000000000..b607fd0a1 --- /dev/null +++ b/internal/httpclient/unlimited_read.go @@ -0,0 +1,195 @@ +package httpclient + +import ( + "errors" + "fmt" + "net/url" + "os" + "strconv" + "strings" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// UNLIMITED READS OVER HTTP. +// +// Every issue listing on the v0 wire spells "unlimited" as `limit=0`, and a +// server started with --allow-non-loopback refuses that spelling outright +// (400, "unlimited reads are loopback-only; pass an explicit limit"). The +// document's recovery is client-side: ask for an explicit limit, and page. So +// this client NEVER puts `limit=0` on the wire, whatever the server's bind +// mode — one spelling for every server is what keeps a loopback-tested caller +// from meeting the refusal for the first time in production. Store.dispatch +// holds the line (refuseUnlimitedOnWire), so a new read path that forgets is a +// loud client error rather than a silent difference between a laptop and a +// fleet. The tripwire guards Store.dispatch only: a caller of the wire.Client +// itself is outside it and must send a positive limit on its own, as the +// down-level claimer's listReadyWork page does (readyClaimPage). +// +// What replaces `limit=0` depends on what paging the operation publishes: +// +// - listIssues publishes a keyset cursor, so an unlimited `bd list` is the +// cursor walk in list_walk.go, at walkPageSize rows a request. No response +// is larger than a page, so the walk is NOT capped here: as on every local +// backend, its only bound is the caller's --max-rows / BEADS_MAX_ROWS. +// - listReadyWork and queryIssues publish NO cursor and no offset (their +// pages are has_more-only by design: a ready set and a predicate query are +// assembled outside the database's keyset order). There is nothing to walk, +// so an unlimited read is ONE request bounded at the cap plus one: the +// overage row is what proves the cap fired. +// +// THE CAP. Every unlimited ready or query read is bounded on the client at +// UnlimitedReadCapEnv rows (DefaultUnlimitedReadCap when unset), because its +// whole answer is one response the server buffers. Past it the read is +// REFUSED with *UnlimitedReadCapError — never truncated. A truncated +// "unlimited" answer is the defect this file exists to remove (the server's +// 100-row ready default used to cap gc's routed-pool queues silently), so the +// only two answers are "every row" and "a typed refusal naming the cap". +// +// SNAPSHOT SEMANTICS, stated per shape because they differ: +// +// - ready and query: one request is one server read, so the answer is one +// consistent snapshot of the ready set / the matching set. +// - list: the cursor walk is one server read PER PAGE, not a snapshot. The +// keyset is strict, so no row is returned twice, and a row that exists and +// matches the filter for the whole walk is returned exactly once. A row +// created during the walk, or one whose status/fields change so that it +// starts or stops matching, may or may not appear — the answer is "the rows +// that matched at the moment each page was read". That is the same promise +// a paging caller of the wire gets, and it is the reason the ready shape +// (whose consumers act on the whole set at once) is NOT emulated through a +// list walk. + +// UnlimitedReadCapEnv bounds how many rows one unlimited ready or query read +// over HTTP may return before it is refused (an unlimited list is a paged +// walk and is not capped). A positive integer; unset means +// DefaultUnlimitedReadCap. A malformed value is an error on every unlimited +// read rather than a silent fall back to the default, so a typo cannot quietly +// re-impose a bound the operator meant to raise. +const UnlimitedReadCapEnv = "BEADS_HTTP_UNLIMITED_READ_CAP" + +// DefaultUnlimitedReadCap is the cap when UnlimitedReadCapEnv is unset. +const DefaultUnlimitedReadCap = 10000 + +// maxUnlimitedReadCap is the largest cap UnlimitedReadCapEnv accepts. It is a +// sanity bound on the operator's knob, not a tuning value: the request asks +// the server to buffer cap+1 rows in one response. +const maxUnlimitedReadCap = 1_000_000 + +// ErrUnlimitedReadCap is the sentinel every *UnlimitedReadCapError matches +// under errors.Is. +var ErrUnlimitedReadCap = errors.New("unlimited read exceeds the http client cap") + +// UnlimitedReadCapError refuses an unlimited ready or query read whose answer +// is larger than the client cap, or whose server would not hand back the +// whole answer. +type UnlimitedReadCapError struct { + // Op is the wire operation (listReadyWork or queryIssues). + Op string + // Cap is the bound in force. + Cap int + // Found is how many rows had arrived when the refusal fired: Cap+1 when the + // cap fired, or the page length when the server truncated (see below). + Found int + // ServerTruncated marks the other refusal: the server returned no more than + // the cap yet reported has_more, so it bounded the page below what was + // asked. Returning that page would be the silent truncation this refusal + // exists to prevent. + ServerTruncated bool +} + +func (e *UnlimitedReadCapError) Error() string { + if e.ServerTruncated { + return fmt.Sprintf("unlimited %s over http: the server returned %d rows with more remaining, "+ + "below the requested bound of %d; refusing a truncated answer (narrow the query)", + e.Op, e.Found, e.Cap+1) + } + return fmt.Sprintf("unlimited %s over http: more than %d rows match (%s=%d); "+ + "narrow the query or raise %s", + e.Op, e.Cap, UnlimitedReadCapEnv, e.Cap, UnlimitedReadCapEnv) +} + +func (e *UnlimitedReadCapError) Unwrap() error { return ErrUnlimitedReadCap } + +// unlimitedReadCap reads UnlimitedReadCapEnv. +func unlimitedReadCap() (int, error) { + raw := strings.TrimSpace(os.Getenv(UnlimitedReadCapEnv)) + if raw == "" { + return DefaultUnlimitedReadCap, nil + } + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > maxUnlimitedReadCap { + return 0, fmt.Errorf("%s=%q: want an integer between 1 and %d", UnlimitedReadCapEnv, raw, maxUnlimitedReadCap) + } + return n, nil +} + +// boundUnlimitedPage writes the bounded `limit` that replaces an unlimited +// read on a cursorless listing, returning the cap in force, or 0 when the +// request is not unlimited and goes out unchanged. +// +// params is the encoder's output and is modified in place. The encoder maps +// fields to the wire's own spelling (`limit=0` for an explicit zero, nothing +// for a legacy zero); how an unlimited read is honored is the pager's +// decision, made here. +func boundUnlimitedPage(params url.Values, unlimited bool) (int, error) { + if !unlimited { + return 0, nil + } + readCap, err := unlimitedReadCap() + if err != nil { + return 0, err + } + params.Set("limit", strconv.Itoa(readCap+1)) + return readCap, nil +} + +// tightenUnlimitedPage lets a caller's MaxRows size an unlimited read's +// request when it is the tighter bound, returning the bound now in force. +// Past MaxRows the answer is refused either way, so sending cap+1 would make +// the server buffer up to the cap's rows only for the caller to throw them +// away. The caller's own MaxRows check must run before checkUnlimitedPage, so +// an answer over it keeps its *ErrTooManyRows attribution. +func tightenUnlimitedPage(params url.Values, readCap, maxRows int) int { + if readCap == 0 || maxRows <= 0 || maxRows >= readCap { + return readCap + } + params.Set("limit", strconv.Itoa(maxRows+1)) + return maxRows +} + +// checkUnlimitedPage applies the cap to the one page a cursorless unlimited +// read received. readCap 0 means the read was bounded by the caller and the +// page is returned as the server shaped it. +func checkUnlimitedPage(op string, readCap, rows int, hasMore bool) error { + switch { + case readCap == 0: + return nil + case rows > readCap: + return &UnlimitedReadCapError{Op: op, Cap: readCap, Found: rows} + case hasMore: + return &UnlimitedReadCapError{Op: op, Cap: readCap, Found: rows, ServerTruncated: true} + } + return nil +} + +// errUnlimitedOnWire is the dispatch tripwire's answer: a client bug, never a +// server condition, so it names no server and is not retried. +var errUnlimitedOnWire = errors.New("httpclient: refusing to send limit=0 (unlimited) on the wire; an unlimited read must be paged or bounded by the client cap") + +// unlimitedSpellingOps are the operations whose `limit=0` means unlimited and +// is refused off loopback. +var unlimitedSpellingOps = map[string]bool{ + wire.OpListReadyWork: true, + wire.OpListIssues: true, + wire.OpQueryIssues: true, +} + +// refuseUnlimitedOnWire is dispatch's guard that `limit=0` never leaves this +// client on an operation that reads it as unlimited. +func refuseUnlimitedOnWire(req wire.Request) error { + if unlimitedSpellingOps[req.Op] && req.Query != nil && req.Query.Get("limit") == "0" { + return fmt.Errorf("%w (%s)", errUnlimitedOnWire, req.Op) + } + return nil +} diff --git a/internal/httpclient/unlimited_read_test.go b/internal/httpclient/unlimited_read_test.go new file mode 100644 index 000000000..b5a842094 --- /dev/null +++ b/internal/httpclient/unlimited_read_test.go @@ -0,0 +1,128 @@ +package httpclient + +import ( + "context" + "errors" + "net/http" + "net/url" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// TestDispatchRefusesUnlimitedOnTheWire pins the tripwire: `limit=0` on an +// operation that reads it as unlimited never leaves dispatch, and the +// refusal is made before any transport is consulted. +// +// GOES RED when refuseUnlimitedOnWire is removed from dispatch. +func TestDispatchRefusesUnlimitedOnTheWire(t *testing.T) { + s := New(Target{BaseURL: &url.URL{Scheme: "http", Host: "127.0.0.1:1"}}, failingWire{t: t}, nil) + for _, op := range []string{wire.OpListReadyWork, wire.OpListIssues, wire.OpQueryIssues} { + err := s.dispatch(t.Context(), wire.Request{ + Op: op, Method: http.MethodGet, Path: "/x", Query: url.Values{"limit": {"0"}}, + }, nil) + if !errors.Is(err, errUnlimitedOnWire) { + t.Errorf("%s with limit=0: err = %v, want errUnlimitedOnWire", op, err) + } + } + // A bounded limit, and an operation whose limit means something else, pass. + for _, req := range []wire.Request{ + {Op: wire.OpListIssues, Query: url.Values{"limit": {"10"}}}, + {Op: wire.OpCountReadyWork, Query: url.Values{"limit": {"0"}}}, + } { + if err := refuseUnlimitedOnWire(req); err != nil { + t.Errorf("%s %v: unexpected refusal %v", req.Op, req.Query, err) + } + } +} + +// failingWire fails the test if dispatch ever reaches it. +type failingWire struct { + WireClient + t *testing.T +} + +func (w failingWire) Preflight(_ context.Context, op string) error { + w.t.Errorf("dispatch reached Preflight for %s; the tripwire must fire first", op) + return errors.New("unreachable") +} + +// TestCheckUnlimitedPage covers the two refusals of a cursorless unlimited +// read, including the one no served fixture can produce: a server that +// truncates below the requested bound and says so with has_more. A full +// cap+1 page that also reports has_more is how a real workspace overflows +// the cap, and it is the cap's refusal, not a truncation. +func TestCheckUnlimitedPage(t *testing.T) { + for _, tc := range []struct { + name string + readCap int + rows int + hasMore bool + truncated bool + refused bool + }{ + {"bounded read passes as shaped", 0, 5, true, false, false}, + {"within the cap", 10, 10, false, false, false}, + {"over the cap", 10, 11, false, false, true}, + {"over the cap with more remaining", 10, 11, true, false, true}, + {"server truncated below the bound", 10, 4, true, true, true}, + } { + err := checkUnlimitedPage(wire.OpQueryIssues, tc.readCap, tc.rows, tc.hasMore) + var capErr *UnlimitedReadCapError + if got := errors.As(err, &capErr); got != tc.refused { + t.Fatalf("%s: refused = %v (%v), want %v", tc.name, got, err, tc.refused) + } + if tc.refused && capErr.ServerTruncated != tc.truncated { + t.Errorf("%s: ServerTruncated = %v, want %v", tc.name, capErr.ServerTruncated, tc.truncated) + } + } +} + +// TestTightenUnlimitedPage pins when a caller's MaxRows sizes the request of +// an unlimited read: only when it is tighter than the cap. A bounded read +// (readCap 0) goes out exactly as the caller shaped it. +func TestTightenUnlimitedPage(t *testing.T) { + for _, tc := range []struct { + name string + readCap int + maxRows int + sent string // the limit before the call + wantBound int + wantLimit string + }{ + {"bounded read", 0, 5, "7", 0, "7"}, + {"no MaxRows", 10, 0, "11", 10, "11"}, + {"MaxRows above the cap", 10, 50, "11", 10, "11"}, + {"MaxRows at the cap", 10, 10, "11", 10, "11"}, + {"MaxRows below the cap", 10, 5, "11", 5, "6"}, + } { + params := url.Values{"limit": {tc.sent}} + if got := tightenUnlimitedPage(params, tc.readCap, tc.maxRows); got != tc.wantBound { + t.Errorf("%s: bound = %d, want %d", tc.name, got, tc.wantBound) + } + if got := params.Get("limit"); got != tc.wantLimit { + t.Errorf("%s: limit = %q, want %q", tc.name, got, tc.wantLimit) + } + } +} + +func TestUnlimitedReadCapEnv(t *testing.T) { + t.Setenv(UnlimitedReadCapEnv, "") + if n, err := unlimitedReadCap(); err != nil || n != DefaultUnlimitedReadCap { + t.Errorf("unset: %d, %v; want the default %d", n, err, DefaultUnlimitedReadCap) + } + t.Setenv(UnlimitedReadCapEnv, " 250 ") + if n, err := unlimitedReadCap(); err != nil || n != 250 { + t.Errorf("250: %d, %v", n, err) + } + t.Setenv(UnlimitedReadCapEnv, "1000000") + if n, err := unlimitedReadCap(); err != nil || n != maxUnlimitedReadCap { + t.Errorf("the inclusive maximum: %d, %v; want %d", n, err, maxUnlimitedReadCap) + } + for _, bad := range []string{"0", "-1", "ten", "1000001"} { + t.Setenv(UnlimitedReadCapEnv, bad) + if _, err := unlimitedReadCap(); err == nil { + t.Errorf("%q was accepted", bad) + } + } +} diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 724bbfda4..5c24bb39f 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -2,6 +2,7 @@ package externaldeps import ( "context" + "errors" "fmt" "slices" "sort" @@ -226,7 +227,7 @@ func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*t queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) issues, err := s.inner.GetReadyWork(ctx, queryFilter) if err != nil { - return nil, err + return nil, innerCapError(err, filter, blockedIDs) } kept := dropBlockedIssues(issues, blockedIDs, filter.Limit) if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { @@ -251,7 +252,7 @@ func (s *Store) readyWorkWithCounts(ctx context.Context, filter types.WorkFilter queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) rows, err := s.inner.GetReadyWorkWithCounts(ctx, queryFilter) if err != nil { - return nil, err + return nil, innerCapError(err, filter, blockedIDs) } kept, _ := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { @@ -291,13 +292,21 @@ func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types // decorator has to drop. Whichever path is taken, the policy itself is never // skipped — only where the exclusion happens differs. // -// The client-side path also takes the MaxRows cap off the inner query. That -// store caps the rows it fetched, which the bump lets outnumber Limit, so +// The client-side path also takes the MaxRows cap off a limited inner query. +// That store caps the rows it fetched, which the bump lets outnumber Limit, so // `--limit N --max-rows N` failed with exit 2 as soon as more than N issues // were ready, refusing a page that could never exceed N. The caller enforces // the cap on the rows it keeps instead (capKeptRows), after the Limit trim, // which is where issueops enforces it on the rows a SQL store delivers // (finishReadyWorkWithCounts). +// +// An unlimited inner query keeps its cap, bumped the same way to MaxRows plus +// len(blockedIDs). Past that many rows, more than MaxRows of them are kept +// whichever rows the drop takes, so the bump refuses nothing the caller's cap +// would admit, and a store that sizes an unlimited request by MaxRows (the +// http bridge, under its client read cap) asks for that bound and one row +// more rather than the whole ready set. Its refusal reaches the caller as the +// caller's own (innerCapError). func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][]string) (queryFilter types.WorkFilter, blockedIDs map[string]bool) { if len(refsByIssue) == 0 { return filter, nil @@ -310,17 +319,21 @@ func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][ blockedIDs[issueID] = true } queryFilter = filter - if queryFilter.Limit > 0 { + switch { + case filter.Limit > 0: queryFilter.Limit += len(blockedIDs) + queryFilter.MaxRows = 0 + queryFilter.MaxRowsSource = "" + case filter.MaxRows > 0: + queryFilter.MaxRows += len(blockedIDs) } - queryFilter.MaxRows = 0 - queryFilter.MaxRowsSource = "" return queryFilter, blockedIDs } -// capKeptRows enforces the MaxRows cap readyExclusion took off the inner -// query, on the kept rows the caller is handed. With nothing to drop -// client-side the inner query kept the cap and has already enforced it. +// capKeptRows enforces the caller's MaxRows cap on the kept rows it is +// handed: readyExclusion took that cap off the inner query, or bumped it by +// the rows the caller drops. With nothing to drop client-side the inner query +// kept the cap and has already enforced it. func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) error { if len(blockedIDs) == 0 { return nil @@ -328,6 +341,22 @@ func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) return issueops.EnforceMaxRowsCap(kept, filter.MaxRows, filter.MaxRowsSource) } +// innerCapError hands the caller its own cap's refusal when the inner query +// refused the MaxRows readyExclusion bumped: Found rows, more than MaxRows +// plus len(blockedIDs), of which at least Found-len(blockedIDs), more than +// MaxRows, would have been kept. Any other error is returned as it is. +func innerCapError(err error, filter types.WorkFilter, blockedIDs map[string]bool) error { + var tooMany *issueops.ErrTooManyRows + if len(blockedIDs) == 0 || filter.Limit > 0 || filter.MaxRows <= 0 || !errors.As(err, &tooMany) { + return err + } + return &issueops.ErrTooManyRows{ + Found: tooMany.Found - len(blockedIDs), + Cap: filter.MaxRows, + Source: filter.MaxRowsSource, + } +} + // dropBlockedIssues is readyExclusion's client-side half: it drops the rows // blockedIDs names and trims what remains to limit. It copies into a fresh // slice rather than compacting issues in place, so the slice s.inner returned @@ -391,7 +420,7 @@ func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.Wo queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) rows, total, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, queryFilter) if err != nil { - return nil, 0, err + return nil, 0, innerCapError(err, filter, blockedIDs) } kept, blockedSeen := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) if len(blockedIDs) == 0 { @@ -407,14 +436,14 @@ func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.Wo // entire ready set, so every blocked row the window contains is every // blocked row the ready set contains: the subtraction is exact. This // is judged from the rows that came back, not from the Limit asked - // for: a zero Limit means every row to a SQL store, but the http wire - // omits it and the server answers with its default page - // (workapi.DefaultReadyLimit), so an unlimited fetch can come back - // truncated too. + // for, so it holds however a store reads a zero Limit: every row to a + // SQL store, and over http one request bounded at the client cap plus + // one, refused rather than truncated past it (an older http client + // sent no limit at all and got the server's default page back). adjustedTotal = total - blockedSeen } else { // The ready set is bigger than the window that came back (the bumped - // window, or the server's default page for an unlimited fetch), so + // window: the caller's page, or CountReadyWork's one-row page), so // the window cannot say how many of its unseen rows are blocked. Nor // can refsByIssue: it names every workspace issue with an unsatisfied // external blocker, closed ones and ones the filter excludes included, @@ -471,8 +500,16 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in } // s.inner has no indexed counter (httpclient.Store among others): reuse // the counts-and-total path above, which already copes with a store whose - // transport cannot express ExcludeIDs at all, nor this zero Limit. - _, total, err := s.readyWorkWithCountsAndTotal(ctx, filter, state) + // transport cannot express ExcludeIDs at all. It asks for a ONE-ROW Lite + // page, never every row: the total is the inner store's own count of the + // whole ready set (over http, the server's countReadyWork beside that + // page), and refetchedReadyTotal sizes the externally blocked case. A zero + // Limit would download the whole ready set to report one number — over + // http, up to BEADS_HTTP_UNLIMITED_READ_CAP full rows, refused past it. + page := filter + page.Limit = 1 + page.Lite = true + _, total, err := s.readyWorkWithCountsAndTotal(ctx, page, state) return total, err } diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index 3909275ed..c5eaf52c9 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -34,10 +34,15 @@ type fakeStore struct { excludeIDsUnsupported bool // defaultPage, when positive, is how many rows a zero-Limit ready listing // returns, while GetReadyWorkWithCountsAndTotal's total still counts the - // whole set: httpclient.Store's shape, whose wire omits a zero limit so - // the server answers with its default page. Zero keeps the SQL stores' - // meaning, every row. + // whole set: the shape httpclient.Store had while its wire omitted a zero + // limit and the server answered with its default page. That store now + // bounds an unlimited read itself, but the decorator still judges a window + // by the rows that came back, which this keeps pinned. Zero keeps the SQL + // stores' meaning, every row. defaultPage int + // readyFilters records every GetReadyWorkWithCountsAndTotal filter, in + // order. + readyFilters []types.WorkFilter } func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } @@ -124,6 +129,7 @@ func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.Wor } func (f *fakeStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + f.readyFilters = append(f.readyFilters, filter) items, err := f.GetReadyWorkWithCounts(ctx, filter) if err != nil { return nil, 0, err @@ -606,13 +612,64 @@ func TestReadyTotalsWhenStoreCannotExpressExcludeIDs(t *testing.T) { } } +// TestCountReadyWorkFallbackFetchesABoundedPage pins what CountReadyWork asks +// of a store with no ReadyWorkCounter: a one-row Lite page whose total is the +// store's own count, never a zero Limit. Over http a zero Limit is an +// unlimited read, which would download every ready row to report one number +// and be refused past BEADS_HTTP_UNLIMITED_READ_CAP. +func TestCountReadyWorkFallbackFetchesABoundedPage(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + for _, tc := range []struct { + name string + deps map[string][]*types.Dependency + want int + // wantFirst is the first fetch's Limit: the one-row page, bumped by + // every externally blocked id, since this store drops them itself. + wantFirst int + }{ + {name: "nothing externally blocked", want: 3, wantFirst: 1}, + { + name: "one issue externally blocked", + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + want: 2, + wantFirst: 2, + }, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &fakeStore{ready: []*types.Issue{a, b, c}, deps: tc.deps, excludeIDsUnsupported: true} + got, err := testStore(raw, &fakeStore{}, true).CountReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("CountReadyWork: %v", err) + } + if got != tc.want { + t.Fatalf("ready count = %d, want %d", got, tc.want) + } + if len(raw.readyFilters) == 0 { + t.Fatal("CountReadyWork fetched nothing; the fallback is not on the path") + } + if first := raw.readyFilters[0]; first.Limit != tc.wantFirst || !first.Lite { + t.Errorf("first fetch: Limit %d, Lite %v; want Limit %d, Lite true", first.Limit, first.Lite, tc.wantFirst) + } + for i, f := range raw.readyFilters { + if f.Limit <= 0 { + t.Errorf("fetch %d asked for every row (Limit %d)", i, f.Limit) + } + } + }) + } +} + // TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage pins the shape -// httpclient.Store has and an every-row Limit==0 fake hid: the wire omits a -// zero Limit, the server answers with its default page, and the total still -// counts the whole ready set. CountReadyWork, text-mode bd ready's total, -// always fetches with Limit 0, so be-z (ready, externally blocked, and past -// that page) once stayed in its total because the decorator took a zero Limit -// for a window that held every row. +// httpclient.Store once had and an every-row Limit==0 fake hid: the wire +// omitted a zero Limit, the server answered with its default page, and the +// total still counted the whole ready set. CountReadyWork, text-mode bd +// ready's total, fetched with Limit 0 then, so be-z (ready, externally +// blocked, and past that page) stayed in its total because the decorator took +// a zero Limit for a window that held every row. The count now asks for a +// one-row page and the JSON total below still asks with Limit 0: both must +// leave be-z out. func TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage(t *testing.T) { const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit ready := make([]*types.Issue, 0, defaultPage+2) diff --git a/issueops/querier.go b/issueops/querier.go index edd3b22b1..c02bf789b 100644 --- a/issueops/querier.go +++ b/issueops/querier.go @@ -61,16 +61,21 @@ type QueryRequest struct { // than a synonym for unlimited: two spellings of "no bound" is one more // than a caller can check for. // - // NOTHING BACKSTOPS IT. ListRequest pairs its Limit with MaxRows, a - // circuit breaker that refuses an oversized answer outright; this request - // has no twin to it by design, and neither knob that sets that one - // (`--max-rows`, BEADS_MAX_ROWS) is read anywhere on the query path. So an - // unlimited query is unbounded in the plain sense: it reads every row the - // expression matches, however many that is, and a caller who asked for one - // gets it rather than an ErrTooManyRows. The ceiling here is Limit itself. - // This is reachable without asking: a piped `bd query` resolves its - // unflagged limit to 0 before the request, the same policy `bd list` has - // applied to piped output since GH#4094. + // NOTHING BACKSTOPS IT on any backend but http (below). ListRequest pairs + // its Limit with MaxRows, a circuit breaker that refuses an oversized + // answer outright; this request has no twin to it by design, and neither + // knob that sets that one (`--max-rows`, BEADS_MAX_ROWS) is read anywhere + // on the query path. So an unlimited query is unbounded in the plain sense: + // it reads every row the expression matches, however many that is, and a + // caller who asked for one gets it rather than an ErrTooManyRows. The + // ceiling here is Limit itself. This is reachable without asking: a piped + // `bd query` resolves its unflagged limit to 0 before the request, the same + // policy `bd list` has applied to piped output since GH#4094. + // + // The http backend is the one exception, because its whole answer is one + // response: it bounds an unlimited query at BEADS_HTTP_UNLIMITED_READ_CAP + // rows and refuses a larger answer with httpclient.ErrUnlimitedReadCap + // rather than truncating it. Limit *int // Offset skips the first N MATCHING rows, on EVERY implementation and for diff --git a/issueops/reader.go b/issueops/reader.go index 3a7d3b0ce..dd761270c 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -93,7 +93,9 @@ type ReadyRequest struct { // Limit bounds the page. Nil means the shared ready default; 0 means // unlimited. It is a pointer so that "unset" and "explicitly unlimited" // stay distinguishable, which is what lets one constant serve both - // surfaces. + // surfaces. Over the http backend an unlimited ready read is one + // response, bounded at BEADS_HTTP_UNLIMITED_READ_CAP rows and refused + // past it with httpclient.ErrUnlimitedReadCap, never truncated. Limit *int // Offset skips the first N matching rows, on EVERY implementation. The // page a caller receives is the rows at positions [Offset, Offset+Limit)