`bd ready --json` had two O(N) hotspots on main. The counts mega-query's reverse-blocker `rc` subquery keys on COALESCE(depends_on_issue_id, …), an expression the pure-Go GMS analyzer cannot auto-index, so it re-scanned its whole materialization once per candidate — O(candidates × blockers). And the "Showing X of N" total re-ran that entire mega-query UNBOUNDED just to size N. Changes: - Generalize one counts builder: sqlbuild.SearchCountsSQL now takes an optional `ids []string`. Empty ids = the predicate form (unchanged callers); non-empty ids = the page-pushed form that constrains the driver and every count subquery to the page, bounding rc by the page instead of the full graph. Removes the drifted SearchCountsSQL/SearchCountsByIDsSQL duplication; the projection order is defined once and DepTargetExpr is used throughout. - issueops.runReadyCountsInTx resolves the ready-ID page with the cheap indexed ID query, then hydrates counts constrained to those IDs, CHUNKED by sqlbuild.QueryBatchSize so large pages stay within backend placeholder limits (no quadratic fallback, no wasted ID pass). The page order is restored so the result is byte-identical to the unbounded ORDER BY … LIMIT. - issueops.CountReadyWorkInTx sizes N with indexed COUNT(*)s over the ready predicate for the issues AND wisps families, minus the wisp-wins overlap — identical to len(GetReadyWorkWithCounts(Limit=0)) even when wisps exist, with no mega-query fallback. - Add ReadyWorkCounter to internal/storage beside the other optional capabilities; DoltStore and EmbeddedDoltStore implement CountReadyWork. cmd/bd/ready.go zeroes the limit at the call site and unwraps to the capability via storage.UnwrapStore, falling back to the unbounded query only when a store predates it. - Extract the shared scan/dedupe loop (scanCountsRowsInTx). - Conformance tests: page counts byte-identical to unbounded, and CountReadyWork == len(unbounded), with fixtures for wisps and a page that spans the chunk boundary. Co-authored-by: Test User <test@test.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
177 lines
6.2 KiB
Go
177 lines
6.2 KiB
Go
package sqlbuild
|
|
|
|
import "fmt"
|
|
|
|
// ReadyWorkIssueColumns is IssueSelectColumns qualified with the "i." alias
|
|
// used by the counts mega-query.
|
|
var ReadyWorkIssueColumns = QualifyColumns(IssueSelectColumns, "i.")
|
|
|
|
// DepJSONObject renders one dependency row as JSON for JSON_ARRAYAGG
|
|
// aggregation in the counts mega-query. Field names must match the JSON tags
|
|
// of types.Dependency.
|
|
const DepJSONObject = `JSON_OBJECT(
|
|
'issue_id', issue_id,
|
|
'depends_on_id', COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external),
|
|
'type', type,
|
|
'created_at', DATE_FORMAT(created_at, '%Y-%m-%dT%H:%i:%sZ'),
|
|
'created_by', created_by,
|
|
'metadata', CAST(metadata AS CHAR),
|
|
'thread_id', thread_id
|
|
)`
|
|
|
|
// SearchCountsSQL renders the counts mega-query: full issue rows aliased "i"
|
|
// plus labels JSON, dep/rdep/comment counts, parent ID, and dependency JSON,
|
|
// for one table family. The scan side is issueops.ScanReadyWorkRowWithCounts,
|
|
// which scans IssueSelectColumns positionally followed by the six extra
|
|
// columns in the order projected here.
|
|
//
|
|
// There are two forms of the same projection, selected by ids:
|
|
//
|
|
// - Predicate form (ids empty): the driver is bounded by whereSQL/orderBySQL/
|
|
// limitSQL and each count subquery aggregates its whole side table. The
|
|
// caller supplies its own args; the returned args slice is nil.
|
|
//
|
|
// - By-IDs form (ids non-empty): whereSQL/orderBySQL/limitSQL are ignored and
|
|
// the driver AND every count subquery are constrained to ids. This keeps
|
|
// each subquery from scanning its whole side table and, crucially, keeps
|
|
// the reverse-blocker rc self-join — whose COALESCE(...) key the pure-Go
|
|
// GMS analyzer cannot auto-index — from re-scanning its full materialization
|
|
// once per driver row. The returned args are the ids repeated once per
|
|
// injection point, in left-to-right placeholder order.
|
|
//
|
|
// The by-IDs form is result-identical to the predicate form over the same set
|
|
// of surviving rows: each per-issue count is a function of the whole dependency
|
|
// graph restricted to that issue, so filtering a subquery's input to ids cannot
|
|
// change any surviving row's count. dep/comment/parent counts are
|
|
// order-insensitive; labels are re-sorted in Go (ScanReadyWorkRowWithCounts).
|
|
// deps_json's element order is whatever JSON_ARRAYAGG emits (SQL does not
|
|
// guarantee one), but it is the same in both forms: restricting the aggregate's
|
|
// input to ids drops only rows for other issues, so the per-issue rows it
|
|
// aggregates — and their relative order — are unchanged.
|
|
//
|
|
// The reverse-blocker rc subquery unions wisp_dependencies only when the caller
|
|
// has probed that the table exists (includeWispReverseDeps).
|
|
func SearchCountsSQL(tables FilterTables, ids []string, whereSQL, orderBySQL, limitSQL string, includeWispReverseDeps, skipLabels bool) (string, []any) {
|
|
byIDs := len(ids) > 0
|
|
inSQL, idArgs := InPlaceholders(ids)
|
|
|
|
// Per-subquery id constraints. Empty strings in the predicate form leave the
|
|
// projection unchanged; in the by-IDs form they push the page down so no
|
|
// subquery aggregates more than the page's worth of rows.
|
|
var labelWhere, depBlocksExtra, rcDepExtra, rcWispExtra, ccWhere, pcExtra, depWhere string
|
|
if byIDs {
|
|
labelWhere = fmt.Sprintf("WHERE issue_id IN (%s)", inSQL)
|
|
depBlocksExtra = fmt.Sprintf(" AND issue_id IN (%s)", inSQL)
|
|
rcDepExtra = fmt.Sprintf(" AND %s IN (%s)", DepTargetExpr, inSQL)
|
|
rcWispExtra = fmt.Sprintf(" AND %s IN (%s)", DepTargetExpr, inSQL)
|
|
ccWhere = fmt.Sprintf("WHERE issue_id IN (%s)", inSQL)
|
|
pcExtra = fmt.Sprintf(" AND issue_id IN (%s)", inSQL)
|
|
depWhere = fmt.Sprintf("WHERE issue_id IN (%s)", inSQL)
|
|
}
|
|
|
|
reverseBlockerSelect := fmt.Sprintf(`
|
|
SELECT %s AS dep_id
|
|
FROM dependencies WHERE type = 'blocks'%s
|
|
`, DepTargetExpr, rcDepExtra)
|
|
if includeWispReverseDeps {
|
|
reverseBlockerSelect += fmt.Sprintf(`
|
|
UNION ALL
|
|
SELECT %s AS dep_id
|
|
FROM wisp_dependencies WHERE type = 'blocks'%s
|
|
`, DepTargetExpr, rcWispExtra)
|
|
}
|
|
|
|
labelsSelect := "l.labels_json AS labels_json"
|
|
labelsJoin := fmt.Sprintf(`
|
|
LEFT JOIN (
|
|
SELECT issue_id, JSON_ARRAYAGG(label) AS labels_json
|
|
FROM %s
|
|
%s
|
|
GROUP BY issue_id
|
|
) l ON l.issue_id = i.id`, tables.Labels, labelWhere)
|
|
if skipLabels {
|
|
labelsSelect = "NULL AS labels_json"
|
|
labelsJoin = ""
|
|
}
|
|
|
|
outerClause := fmt.Sprintf("%s\n\t\t%s\n\t\t%s", whereSQL, orderBySQL, limitSQL)
|
|
if byIDs {
|
|
outerClause = fmt.Sprintf("WHERE i.id IN (%s)", inSQL)
|
|
}
|
|
|
|
sqlText := fmt.Sprintf(`
|
|
SELECT %s,
|
|
%s,
|
|
COALESCE(dc.cnt, 0) AS dep_count,
|
|
COALESCE(rc.cnt, 0) AS rdep_count,
|
|
COALESCE(cc.cnt, 0) AS comment_count,
|
|
pc.parent_id AS parent_id,
|
|
d.deps_json AS deps_json
|
|
FROM %s i
|
|
%s
|
|
LEFT JOIN (
|
|
SELECT issue_id, COUNT(*) AS cnt
|
|
FROM %s
|
|
WHERE type = 'blocks'%s
|
|
GROUP BY issue_id
|
|
) dc ON dc.issue_id = i.id
|
|
LEFT JOIN (
|
|
SELECT dep_id, COUNT(*) AS cnt FROM (
|
|
%s
|
|
) all_blockers GROUP BY dep_id
|
|
) rc ON rc.dep_id = i.id
|
|
LEFT JOIN (
|
|
SELECT issue_id, COUNT(*) AS cnt
|
|
FROM %s
|
|
%s
|
|
GROUP BY issue_id
|
|
) cc ON cc.issue_id = i.id
|
|
LEFT JOIN (
|
|
SELECT issue_id,
|
|
MIN(%s) AS parent_id
|
|
FROM %s
|
|
WHERE type = 'parent-child'%s
|
|
GROUP BY issue_id
|
|
) pc ON pc.issue_id = i.id
|
|
LEFT JOIN (
|
|
SELECT issue_id, JSON_ARRAYAGG(%s) AS deps_json
|
|
FROM %s
|
|
%s
|
|
GROUP BY issue_id
|
|
) d ON d.issue_id = i.id
|
|
%s
|
|
`,
|
|
ReadyWorkIssueColumns,
|
|
labelsSelect,
|
|
tables.Main,
|
|
labelsJoin,
|
|
tables.Dependencies, depBlocksExtra,
|
|
reverseBlockerSelect,
|
|
tables.Comments, ccWhere,
|
|
DepTargetExpr, tables.Dependencies, pcExtra,
|
|
DepJSONObject, tables.Dependencies, depWhere,
|
|
outerClause,
|
|
)
|
|
|
|
if !byIDs {
|
|
return sqlText, nil
|
|
}
|
|
|
|
// args follow the placeholder order in sqlText: labels join (unless
|
|
// skipped), dc, rc dependencies branch, rc wisp branch (if any), cc, pc, d,
|
|
// then the driver.
|
|
args := make([]any, 0, len(idArgs)*8)
|
|
if !skipLabels {
|
|
args = append(args, idArgs...)
|
|
}
|
|
args = append(args, idArgs...) // dc
|
|
args = append(args, idArgs...) // rc dependencies
|
|
if includeWispReverseDeps {
|
|
args = append(args, idArgs...) // rc wisp_dependencies
|
|
}
|
|
args = append(args, idArgs...) // cc
|
|
args = append(args, idArgs...) // pc
|
|
args = append(args, idArgs...) // d
|
|
args = append(args, idArgs...) // driver
|
|
return sqlText, args
|
|
}
|