From cf2e0d9a015d923b0cdf68f504d95dce071bec7f Mon Sep 17 00:00:00 2001 From: Jacob Hausler Date: Sun, 26 Jul 2026 00:29:55 -0500 Subject: [PATCH] feat(ready): add pagination metadata to BD_JSON_ENVELOPE=1 output when truncated (#4897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ready): add pagination metadata to BD_JSON_ENVELOPE=1 output when truncated bd ready --json with BD_JSON_ENVELOPE=1 now includes a "pagination" key in the envelope when results are capped by --limit: { "schema_version": 1, "data": [...], "pagination": {"returned": 100, "total": 149, "truncated": true} } Without the envelope (default bare-array mode), the existing stderr text hint ("Showing 100 of 149 ready issues. Use --limit 0 for all...") is unchanged. The pagination key is absent when results are not truncated, so callers can check for its presence / the truncated field rather than comparing returned vs total. Fixes the silent truncation problem for programmatic consumers (gc, agents, sweep scripts) that call bd ready --json and consume the JSON payload from stdout — they cannot read the stderr prose hint, so they received no signal that the list was capped. Closes #4892 * fix(ready): emit pagination metadata on the proxied route too runReadyProxiedServer's JSON branch called the bare outputJSON(results) even though page.HasMore already carried enough information to detect truncation, so BD_JSON_ENVELOPE=1 consumers on the proxied path (the default for non-CGO/proxied builds) had no way to see the pagination signal that the direct route already exposes. Build a PaginationMeta with Returned/Truncated from page.HasMore && in.filter.Limit > 0 and route through outputJSONWithPagination instead; Total is left unset since this backend's domain page has no cheap total-count equivalent. Adds proxied-route coverage (pagination_envelope_truncated and pagination_envelope_not_truncated subtests) exercising the envelope under BD_JSON_ENVELOPE=1 for both truncated and non-truncated results. Closes the blocking item from the 2026-07-23 CHANGES_REQUESTED review on gastownhall/beads#4897. Agent-Signature: claude-sonnet-5-medium on behalf of maphew --------- Co-authored-by: Jacob Hausler Co-authored-by: matt wilkie --- cmd/bd/output.go | 31 ++++++++++- cmd/bd/output_test.go | 63 ++++++++++++++++++++++ cmd/bd/ready.go | 10 +++- cmd/bd/ready_proxied_integration_test.go | 66 ++++++++++++++++++++++++ cmd/bd/ready_proxied_server.go | 16 +++++- 5 files changed, 181 insertions(+), 5 deletions(-) diff --git a/cmd/bd/output.go b/cmd/bd/output.go index 487c60683..ffefa4cfa 100644 --- a/cmd/bd/output.go +++ b/cmd/bd/output.go @@ -11,15 +11,42 @@ import ( const JSONSchemaVersion = 1 +// PaginationMeta carries truncation context for paginated JSON responses. +// It is included in the BD_JSON_ENVELOPE=1 output under the "pagination" key +// whenever the result set was capped by a --limit. +type PaginationMeta struct { + Returned int `json:"returned"` + Total int `json:"total,omitempty"` + Truncated bool `json:"truncated"` +} + func jsonEnvelopeEnabled() bool { return os.Getenv("BD_JSON_ENVELOPE") == "1" } func outputJSON(v interface{}) error { - wrapped := wrapWithSchemaVersion(v) + return outputJSONWithPagination(v, nil) +} + +// outputJSONWithPagination emits v as JSON, optionally including pagination +// metadata. When BD_JSON_ENVELOPE=1 and p is non-nil, the envelope gains a +// "pagination" key so programmatic consumers can detect truncation without +// parsing stderr. When the envelope is not active, p is ignored and the +// existing stderr text hint handles the human/text path. +func outputJSONWithPagination(v interface{}, p *PaginationMeta) error { + var out interface{} + if jsonEnvelopeEnabled() && p != nil { + out = map[string]interface{}{ + "schema_version": JSONSchemaVersion, + "data": v, + "pagination": p, + } + } else { + out = wrapWithSchemaVersion(v) + } encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") - if err := encoder.Encode(wrapped); err != nil { + if err := encoder.Encode(out); err != nil { return fmt.Errorf("encoding JSON: %v", err) } diff --git a/cmd/bd/output_test.go b/cmd/bd/output_test.go index 1292da42b..1fd6e7601 100644 --- a/cmd/bd/output_test.go +++ b/cmd/bd/output_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "testing" ) @@ -140,3 +141,65 @@ func TestWrapWithSchemaVersion_Envelope_RoundTrip(t *testing.T) { t.Errorf("data.count = %v, want 42", innerData["count"]) } } + +func TestOutputJSONWithPagination_NoTruncation(t *testing.T) { + items := []string{"a", "b"} + out := captureStdout(t, func() error { + return outputJSONWithPagination(items, nil) + }) + var arr []string + if err := json.Unmarshal([]byte(bytes.TrimSpace([]byte(out))), &arr); err != nil { + t.Fatalf("unmarshal: %v\noutput: %s", err, out) + } + if len(arr) != 2 { + t.Errorf("len = %d, want 2", len(arr)) + } +} + +func TestOutputJSONWithPagination_EnvelopeTruncated(t *testing.T) { + t.Setenv("BD_JSON_ENVELOPE", "1") + + items := []string{"a", "b", "c"} + pag := &PaginationMeta{Returned: 3, Total: 10, Truncated: true} + out := captureStdout(t, func() error { + return outputJSONWithPagination(items, pag) + }) + + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(bytes.TrimSpace([]byte(out))), &m); err != nil { + t.Fatalf("unmarshal: %v\noutput: %s", err, out) + } + if _, ok := m["pagination"]; !ok { + t.Fatal("missing 'pagination' key in envelope") + } + var got PaginationMeta + if err := json.Unmarshal(m["pagination"], &got); err != nil { + t.Fatalf("unmarshal pagination: %v", err) + } + if !got.Truncated { + t.Error("pagination.truncated = false, want true") + } + if got.Returned != 3 { + t.Errorf("pagination.returned = %d, want 3", got.Returned) + } + if got.Total != 10 { + t.Errorf("pagination.total = %d, want 10", got.Total) + } +} + +func TestOutputJSONWithPagination_EnvelopeNotTruncated(t *testing.T) { + t.Setenv("BD_JSON_ENVELOPE", "1") + + items := []string{"a", "b"} + out := captureStdout(t, func() error { + return outputJSONWithPagination(items, nil) + }) + + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(bytes.TrimSpace([]byte(out))), &m); err != nil { + t.Fatalf("unmarshal: %v\noutput: %s", err, out) + } + if _, ok := m["pagination"]; ok { + t.Error("unexpected 'pagination' key when not truncated") + } +} diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index aa9fa5a43..e8658e6f8 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -297,7 +297,15 @@ This is useful for agents executing molecules to see which steps can run next.`, if results == nil { results = []*types.IssueWithCounts{} } - if jerr := outputJSON(results); jerr != nil { + var pag *PaginationMeta + if truncated { + pag = &PaginationMeta{ + Returned: len(results), + Total: totalReady, + Truncated: true, + } + } + if jerr := outputJSONWithPagination(results, pag); jerr != nil { return jerr } if truncated { diff --git a/cmd/bd/ready_proxied_integration_test.go b/cmd/bd/ready_proxied_integration_test.go index 596c1a35d..4442148d5 100644 --- a/cmd/bd/ready_proxied_integration_test.go +++ b/cmd/bd/ready_proxied_integration_test.go @@ -193,6 +193,72 @@ func TestProxiedServerReady(t *testing.T) { } }) + t.Run("pagination_envelope_truncated", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "rdpgt") + for i := 0; i < 4; i++ { + bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("Pag item %d", i), "--label", "rdpgt") + } + stdout, _, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BD_JSON_ENVELOPE=1"}, + "ready", "--json", "--limit", "2", "--label", "rdpgt") + if err != nil { + t.Fatalf("bd ready --json --limit 2 failed: %v\nstdout: %s", err, stdout) + } + var envelope struct { + Data []*types.IssueWithCounts `json:"data"` + Pagination *PaginationMeta `json:"pagination"` + } + s := strings.TrimSpace(stdout) + start := strings.Index(s, "{") + if start < 0 { + t.Fatalf("no JSON envelope object in stdout: %s", stdout) + } + if err := json.Unmarshal([]byte(s[start:]), &envelope); err != nil { + t.Fatalf("parse envelope JSON: %v\n%s", err, s[start:]) + } + if len(envelope.Data) != 2 { + t.Fatalf("envelope.data length = %d, want 2", len(envelope.Data)) + } + if envelope.Pagination == nil { + t.Fatal("missing 'pagination' key in truncated envelope") + } + if !envelope.Pagination.Truncated { + t.Error("pagination.truncated = false, want true") + } + if envelope.Pagination.Returned != 2 { + t.Errorf("pagination.returned = %d, want 2", envelope.Pagination.Returned) + } + // Total is unavailable on the proxied backend; omitempty must drop + // the key rather than emit a false "total": 0. + if strings.Contains(s[start:], `"total"`) { + t.Errorf("unexpected 'total' key in proxied pagination (unavailable on this backend): %s", s[start:]) + } + }) + + t.Run("pagination_envelope_not_truncated", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "rdpgn") + bdProxiedCreate(t, bd, p.dir, "Pag not truncated", "--label", "rdpgn") + stdout, _, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BD_JSON_ENVELOPE=1"}, + "ready", "--json", "--limit", "10", "--label", "rdpgn") + if err != nil { + t.Fatalf("bd ready --json --limit 10 failed: %v\nstdout: %s", err, stdout) + } + var envelope map[string]json.RawMessage + s := strings.TrimSpace(stdout) + start := strings.Index(s, "{") + if start < 0 { + t.Fatalf("no JSON envelope object in stdout: %s", stdout) + } + if err := json.Unmarshal([]byte(s[start:]), &envelope); err != nil { + t.Fatalf("parse envelope JSON: %v\n%s", err, s[start:]) + } + // Parity with the direct route: no pagination key when not truncated. + if raw, ok := envelope["pagination"]; ok { + t.Errorf("unexpected 'pagination' key when not truncated: %s", raw) + } + }) + t.Run("offset_with_large_finite_limit", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "rdofflz") diff --git a/cmd/bd/ready_proxied_server.go b/cmd/bd/ready_proxied_server.go index 0a963b86f..dd9b434ee 100644 --- a/cmd/bd/ready_proxied_server.go +++ b/cmd/bd/ready_proxied_server.go @@ -104,8 +104,20 @@ func runReadyProxiedList(ctx context.Context, uw uow.UnitOfWork, in readyInput) if results == nil { results = []*types.IssueWithCounts{} } - _ = outputJSON(results) - if page.HasMore && in.filter.Limit > 0 { + truncated := page.HasMore && in.filter.Limit > 0 + // Parity with the direct route: the pagination key is emitted only + // when truncated. Total is unavailable from this backend's domain + // page (no cheap COUNT(*) equivalent on the proxied path), so it is + // left unset (0) unlike the direct route's fully-populated meta. + var pag *PaginationMeta + if truncated { + pag = &PaginationMeta{ + Returned: len(results), + Truncated: true, + } + } + _ = outputJSONWithPagination(results, pag) + if truncated { fmt.Fprintf(os.Stderr, "Showing %d ready issues; more matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.\n", len(results)) } return nil