beads/internal/storage/sqlbuild/counts.go
Julian Knutsen fb527d034a
perf(ready): page-push ready-work counts and size the total with cheap COUNT(*)s (#4679)
`bd ready --json` had two O(N) hotspots on main. The counts mega-query's
reverse-blocker `rc` subquery keys on COALESCE(depends_on_issue_id, …), an
expression the pure-Go GMS analyzer cannot auto-index, so it re-scanned its
whole materialization once per candidate — O(candidates × blockers). And the
"Showing X of N" total re-ran that entire mega-query UNBOUNDED just to size N.

Changes:
- Generalize one counts builder: sqlbuild.SearchCountsSQL now takes an optional
  `ids []string`. Empty ids = the predicate form (unchanged callers); non-empty
  ids = the page-pushed form that constrains the driver and every count
  subquery to the page, bounding rc by the page instead of the full graph.
  Removes the drifted SearchCountsSQL/SearchCountsByIDsSQL duplication; the
  projection order is defined once and DepTargetExpr is used throughout.
- issueops.runReadyCountsInTx resolves the ready-ID page with the cheap indexed
  ID query, then hydrates counts constrained to those IDs, CHUNKED by
  sqlbuild.QueryBatchSize so large pages stay within backend placeholder limits
  (no quadratic fallback, no wasted ID pass). The page order is restored so the
  result is byte-identical to the unbounded ORDER BY … LIMIT.
- issueops.CountReadyWorkInTx sizes N with indexed COUNT(*)s over the ready
  predicate for the issues AND wisps families, minus the wisp-wins overlap —
  identical to len(GetReadyWorkWithCounts(Limit=0)) even when wisps exist, with
  no mega-query fallback.
- Add ReadyWorkCounter to internal/storage beside the other optional
  capabilities; DoltStore and EmbeddedDoltStore implement CountReadyWork.
  cmd/bd/ready.go zeroes the limit at the call site and unwraps to the
  capability via storage.UnwrapStore, falling back to the unbounded query only
  when a store predates it.
- Extract the shared scan/dedupe loop (scanCountsRowsInTx).
- Conformance tests: page counts byte-identical to unbounded, and
  CountReadyWork == len(unbounded), with fixtures for wisps and a page that
  spans the chunk boundary.

Co-authored-by: Test User <test@test.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 18:07:41 -07:00

177 lines
6.2 KiB
Go

package sqlbuild
import "fmt"
// ReadyWorkIssueColumns is IssueSelectColumns qualified with the "i." alias
// used by the counts mega-query.
var ReadyWorkIssueColumns = QualifyColumns(IssueSelectColumns, "i.")
// DepJSONObject renders one dependency row as JSON for JSON_ARRAYAGG
// aggregation in the counts mega-query. Field names must match the JSON tags
// of types.Dependency.
const DepJSONObject = `JSON_OBJECT(
'issue_id', issue_id,
'depends_on_id', COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external),
'type', type,
'created_at', DATE_FORMAT(created_at, '%Y-%m-%dT%H:%i:%sZ'),
'created_by', created_by,
'metadata', CAST(metadata AS CHAR),
'thread_id', thread_id
)`
// SearchCountsSQL renders the counts mega-query: full issue rows aliased "i"
// plus labels JSON, dep/rdep/comment counts, parent ID, and dependency JSON,
// for one table family. The scan side is issueops.ScanReadyWorkRowWithCounts,
// which scans IssueSelectColumns positionally followed by the six extra
// columns in the order projected here.
//
// There are two forms of the same projection, selected by ids:
//
// - Predicate form (ids empty): the driver is bounded by whereSQL/orderBySQL/
// limitSQL and each count subquery aggregates its whole side table. The
// caller supplies its own args; the returned args slice is nil.
//
// - By-IDs form (ids non-empty): whereSQL/orderBySQL/limitSQL are ignored and
// the driver AND every count subquery are constrained to ids. This keeps
// each subquery from scanning its whole side table and, crucially, keeps
// the reverse-blocker rc self-join — whose COALESCE(...) key the pure-Go
// GMS analyzer cannot auto-index — from re-scanning its full materialization
// once per driver row. The returned args are the ids repeated once per
// injection point, in left-to-right placeholder order.
//
// The by-IDs form is result-identical to the predicate form over the same set
// of surviving rows: each per-issue count is a function of the whole dependency
// graph restricted to that issue, so filtering a subquery's input to ids cannot
// change any surviving row's count. dep/comment/parent counts are
// order-insensitive; labels are re-sorted in Go (ScanReadyWorkRowWithCounts).
// deps_json's element order is whatever JSON_ARRAYAGG emits (SQL does not
// guarantee one), but it is the same in both forms: restricting the aggregate's
// input to ids drops only rows for other issues, so the per-issue rows it
// aggregates — and their relative order — are unchanged.
//
// The reverse-blocker rc subquery unions wisp_dependencies only when the caller
// has probed that the table exists (includeWispReverseDeps).
func SearchCountsSQL(tables FilterTables, ids []string, whereSQL, orderBySQL, limitSQL string, includeWispReverseDeps, skipLabels bool) (string, []any) {
byIDs := len(ids) > 0
inSQL, idArgs := InPlaceholders(ids)
// Per-subquery id constraints. Empty strings in the predicate form leave the
// projection unchanged; in the by-IDs form they push the page down so no
// subquery aggregates more than the page's worth of rows.
var labelWhere, depBlocksExtra, rcDepExtra, rcWispExtra, ccWhere, pcExtra, depWhere string
if byIDs {
labelWhere = fmt.Sprintf("WHERE issue_id IN (%s)", inSQL)
depBlocksExtra = fmt.Sprintf(" AND issue_id IN (%s)", inSQL)
rcDepExtra = fmt.Sprintf(" AND %s IN (%s)", DepTargetExpr, inSQL)
rcWispExtra = fmt.Sprintf(" AND %s IN (%s)", DepTargetExpr, inSQL)
ccWhere = fmt.Sprintf("WHERE issue_id IN (%s)", inSQL)
pcExtra = fmt.Sprintf(" AND issue_id IN (%s)", inSQL)
depWhere = fmt.Sprintf("WHERE issue_id IN (%s)", inSQL)
}
reverseBlockerSelect := fmt.Sprintf(`
SELECT %s AS dep_id
FROM dependencies WHERE type = 'blocks'%s
`, DepTargetExpr, rcDepExtra)
if includeWispReverseDeps {
reverseBlockerSelect += fmt.Sprintf(`
UNION ALL
SELECT %s AS dep_id
FROM wisp_dependencies WHERE type = 'blocks'%s
`, DepTargetExpr, rcWispExtra)
}
labelsSelect := "l.labels_json AS labels_json"
labelsJoin := fmt.Sprintf(`
LEFT JOIN (
SELECT issue_id, JSON_ARRAYAGG(label) AS labels_json
FROM %s
%s
GROUP BY issue_id
) l ON l.issue_id = i.id`, tables.Labels, labelWhere)
if skipLabels {
labelsSelect = "NULL AS labels_json"
labelsJoin = ""
}
outerClause := fmt.Sprintf("%s\n\t\t%s\n\t\t%s", whereSQL, orderBySQL, limitSQL)
if byIDs {
outerClause = fmt.Sprintf("WHERE i.id IN (%s)", inSQL)
}
sqlText := fmt.Sprintf(`
SELECT %s,
%s,
COALESCE(dc.cnt, 0) AS dep_count,
COALESCE(rc.cnt, 0) AS rdep_count,
COALESCE(cc.cnt, 0) AS comment_count,
pc.parent_id AS parent_id,
d.deps_json AS deps_json
FROM %s i
%s
LEFT JOIN (
SELECT issue_id, COUNT(*) AS cnt
FROM %s
WHERE type = 'blocks'%s
GROUP BY issue_id
) dc ON dc.issue_id = i.id
LEFT JOIN (
SELECT dep_id, COUNT(*) AS cnt FROM (
%s
) all_blockers GROUP BY dep_id
) rc ON rc.dep_id = i.id
LEFT JOIN (
SELECT issue_id, COUNT(*) AS cnt
FROM %s
%s
GROUP BY issue_id
) cc ON cc.issue_id = i.id
LEFT JOIN (
SELECT issue_id,
MIN(%s) AS parent_id
FROM %s
WHERE type = 'parent-child'%s
GROUP BY issue_id
) pc ON pc.issue_id = i.id
LEFT JOIN (
SELECT issue_id, JSON_ARRAYAGG(%s) AS deps_json
FROM %s
%s
GROUP BY issue_id
) d ON d.issue_id = i.id
%s
`,
ReadyWorkIssueColumns,
labelsSelect,
tables.Main,
labelsJoin,
tables.Dependencies, depBlocksExtra,
reverseBlockerSelect,
tables.Comments, ccWhere,
DepTargetExpr, tables.Dependencies, pcExtra,
DepJSONObject, tables.Dependencies, depWhere,
outerClause,
)
if !byIDs {
return sqlText, nil
}
// args follow the placeholder order in sqlText: labels join (unless
// skipped), dc, rc dependencies branch, rc wisp branch (if any), cc, pc, d,
// then the driver.
args := make([]any, 0, len(idArgs)*8)
if !skipLabels {
args = append(args, idArgs...)
}
args = append(args, idArgs...) // dc
args = append(args, idArgs...) // rc dependencies
if includeWispReverseDeps {
args = append(args, idArgs...) // rc wisp_dependencies
}
args = append(args, idArgs...) // cc
args = append(args, idArgs...) // pc
args = append(args, idArgs...) // d
args = append(args, idArgs...) // driver
return sqlText, args
}