perf(search): Q2+Q3+Q5 — skip-wisps merge, id-shrunk Pattern B, wisp count cache (bd-qi4qr, bd-co6m6)

Q2 (bd-qi4qr): add IssueFilter.SkipWisps; set it in count/query/list callers that never
need ephemeral results; SearchIssuesInTx returns early before the wisps merge.

Q3 (bd-co6m6): add IssueFilter.NoIDShrink; searchTableInTx dispatches Pattern B
(SELECT id LIMIT n + batch hydrate) when Limit>0 and NoIDShrink is false; Pattern A
(full 47-column scan) retained for unlimited queries and NoIDShrink callers.

Q5 (bd-qi4qr): per-process wisp row-count cache on DoltStore (wispCountCached/
wispCountZero); SearchIssues short-circuits SkipWisps when table is known empty,
avoiding the COUNT(*) hit on every search; cache invalidated on CreateIssue,
CreateIssuesWithFullOptions, DeleteIssue, DeleteIssues, and DemoteToWisp.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Jim Wordelman 2026-05-17 12:33:10 -07:00
commit 9a29a158fd
4 changed files with 149 additions and 31 deletions

View file

@ -197,6 +197,7 @@ Examples:
filter.PriorityMax = &priorityMax
}
filter.SkipWisps = true // bd count never needs ephemeral wisp results
issues, err := store.SearchIssues(ctx, "", filter)
if err != nil {
FatalError("%v", err)

View file

@ -1092,6 +1092,11 @@ var listCmd = &cobra.Command{
FatalError("%v", err)
}
} else {
// Skip wisps merge when the caller doesn't need ephemeral results
// (no --include-infra, not an infra type request). Q2 perf opt.
if !includeInfra && (issueType == "" || !isInfra(issueType)) {
filter.SkipWisps = true
}
var err error
issues, err = activeStore.SearchIssues(ctx, "", filter)
if err != nil {

View file

@ -12,6 +12,9 @@ import (
// SearchIssuesInTx executes a filtered issue search within an existing transaction.
// It queries the issues table, optionally merges wisps, and returns hydrated issues
// with labels populated.
//
// Set filter.SkipWisps=true for callers that never need ephemeral results; this
// avoids the unconditional full-table wisps scan (Q2 perf opt).
func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) ([]*types.Issue, error) {
// Route ephemeral-only queries to wisps table.
if filter.Ephemeral != nil && *filter.Ephemeral {
@ -30,6 +33,11 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type
return nil, fmt.Errorf("search issues: %w", err)
}
// Skip wisps merge entirely when caller opts out (Q2: perf escape hatch).
if filter.SkipWisps {
return results, nil
}
// When filter.Ephemeral is nil (search everything) or false (non-ephemeral
// only), also search the wisps table and merge results. NoHistory beads are
// stored in the wisps table with ephemeral=0, so they must survive an
@ -67,6 +75,11 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type
}
// searchTableInTx runs a filtered search against a specific table set (issues or wisps).
//
// When filter.Limit > 0 and !filter.NoIDShrink, uses Pattern B (id-shrunk): a cheap
// SELECT id scan + batch hydration instead of a full 47-column projection scan.
// Pattern B is equivalent to Pattern A but faster on large corpora where most rows
// are never needed (mirrors the pattern in scanIssueIDs and GetStaleIssuesInTx).
func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables) ([]*types.Issue, error) {
fromSQL, labelWhere, labelArgs, labelDriven, filterForClauses := buildLabelDrivenSearch(filter, tables)
whereClauses, args, err := BuildIssueFilterClauses(query, filterForClauses, tables)
@ -83,6 +96,12 @@ func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types
whereSQL = "WHERE " + strings.Join(whereClauses, " AND ")
}
// Pattern B: when Limit > 0, use a cheap id scan then hydrate in batch.
if filter.Limit > 0 && !filter.NoIDShrink {
return searchTablePatternB(ctx, tx, fromSQL, whereSQL, args, filter, tables, labelDriven)
}
// Pattern A: full 47-column scan (used for unlimited queries or when NoIDShrink is set).
limitSQL := ""
if filter.Limit > 0 {
limitSQL = fmt.Sprintf(" LIMIT %d", filter.Limit)
@ -120,44 +139,133 @@ func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types
return nil, fmt.Errorf("search %s: rows: %w", tables.Main, err)
}
// Hydrate labels in bulk after closing the result set, so we don't hold
// multiple active result sets on the same connection.
if len(issues) > 0 {
ids := make([]string, len(issues))
for i, issue := range issues {
ids[i] = issue.ID
}
if !filter.SkipLabels {
// Fast path: searchTableInTx queries exclusively either the issues
// or wisps table, so every ID in `ids` belongs to tables.Labels.
// Skip the per-batch wisp-partition round-trip that the generic
// GetLabelsForIssuesInTx performs (GH#3414).
labelMap, labelErr := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids)
if labelErr != nil {
return nil, fmt.Errorf("search %s: hydrate labels: %w", tables.Main, labelErr)
}
for _, issue := range issues {
if labels, ok := labelMap[issue.ID]; ok {
issue.Labels = labels
}
}
}
if err := hydrateIssues(ctx, tx, issues, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil {
return nil, fmt.Errorf("search %s: hydrate: %w", tables.Main, err)
}
// Optionally hydrate dependencies in bulk (same batched pattern as labels).
if filter.IncludeDependencies {
depMap, depErr := GetDependencyRecordsForIssuesFromTableInTx(ctx, tx, tables.Dependencies, ids)
if depErr != nil {
return nil, fmt.Errorf("search %s: hydrate dependencies: %w", tables.Main, depErr)
}
for _, issue := range issues {
if deps, ok := depMap[issue.ID]; ok {
issue.Dependencies = deps
}
return issues, nil
}
// searchTablePatternB runs Pattern B: SELECT id LIMIT n → batch hydrate.
// Equivalent result to Pattern A but avoids streaming all 47 columns for rows
// that won't survive the LIMIT cut. Mirrors the approach in GetStaleIssuesInTx.
func searchTablePatternB(ctx context.Context, tx *sql.Tx, fromSQL, whereSQL string, args []interface{}, filter types.IssueFilter, tables FilterTables, labelDriven bool) ([]*types.Issue, error) {
idSelect := "SELECT "
if labelDriven {
idSelect = "SELECT DISTINCT "
}
//nolint:gosec // G201: SQL fragments from fixed column/table names and parameterized filters.
idQuery := fmt.Sprintf(`%s%s.id FROM %s %s ORDER BY %s.priority ASC, %s.created_at DESC, %s.id ASC LIMIT %d`,
idSelect, tables.Main, fromSQL, whereSQL, tables.Main, tables.Main, tables.Main, filter.Limit)
rows, err := tx.QueryContext(ctx, idQuery, args...)
if err != nil {
return nil, fmt.Errorf("search %s (id scan): %w", tables.Main, err)
}
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
_ = rows.Close()
return nil, fmt.Errorf("search %s (id scan): scan: %w", tables.Main, err)
}
ids = append(ids, id)
}
_ = rows.Close()
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("search %s (id scan): rows: %w", tables.Main, err)
}
if len(ids) == 0 {
return nil, nil
}
// Batch-fetch full rows from the known table (no wispSet partition needed).
placeholders := make([]string, len(ids))
fetchArgs := make([]interface{}, len(ids))
for i, id := range ids {
placeholders[i] = "?"
fetchArgs[i] = id
}
//nolint:gosec // G201: table name is a fixed constant from FilterTables.
fetchSQL := fmt.Sprintf(`SELECT %s FROM %s WHERE id IN (%s)`,
IssueSelectColumns, tables.Main, strings.Join(placeholders, ","))
fetchRows, err := tx.QueryContext(ctx, fetchSQL, fetchArgs...)
if err != nil {
return nil, fmt.Errorf("search %s (hydrate): %w", tables.Main, err)
}
issueMap := make(map[string]*types.Issue, len(ids))
for fetchRows.Next() {
issue, scanErr := ScanIssueFrom(fetchRows)
if scanErr != nil {
_ = fetchRows.Close()
return nil, fmt.Errorf("search %s (hydrate): scan: %w", tables.Main, scanErr)
}
issueMap[issue.ID] = issue
}
_ = fetchRows.Close()
if err := fetchRows.Err(); err != nil {
return nil, fmt.Errorf("search %s (hydrate): rows: %w", tables.Main, err)
}
// Reorder to preserve the id-scan ORDER BY.
issues := make([]*types.Issue, 0, len(ids))
for _, id := range ids {
if issue, ok := issueMap[id]; ok {
issues = append(issues, issue)
}
}
if err := hydrateIssues(ctx, tx, issues, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil {
return nil, fmt.Errorf("search %s (pattern B): hydrate: %w", tables.Main, err)
}
return issues, nil
}
// hydrateIssues populates labels (and optionally dependencies) on a slice of issues.
// All issues must belong to tables.Main; labels come from tables.Labels.
// When skipLabels is true, label hydration is suppressed (Issue.Labels is left nil).
func hydrateIssues(ctx context.Context, tx *sql.Tx, issues []*types.Issue, tables FilterTables, includeDeps bool, skipLabels bool) error {
if len(issues) == 0 {
return nil
}
ids := make([]string, len(issues))
for i, issue := range issues {
ids[i] = issue.ID
}
if !skipLabels {
// Fast path: every ID in `ids` belongs to tables.Labels.
// Skip the per-batch wisp-partition round-trip (GH#3414).
labelMap, err := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids)
if err != nil {
return fmt.Errorf("hydrate labels: %w", err)
}
for _, issue := range issues {
if labels, ok := labelMap[issue.ID]; ok {
issue.Labels = labels
}
}
}
return issues, nil
if includeDeps {
depMap, err := GetDependencyRecordsForIssuesFromTableInTx(ctx, tx, tables.Dependencies, ids)
if err != nil {
return fmt.Errorf("hydrate dependencies: %w", err)
}
for _, issue := range issues {
if deps, ok := depMap[issue.ID]; ok {
issue.Dependencies = deps
}
}
}
return nil
}
func buildLabelDrivenSearch(filter types.IssueFilter, tables FilterTables) (string, []string, []interface{}, bool, types.IssueFilter) {

View file

@ -1282,6 +1282,10 @@ type IssueFilter struct {
// skipped and Issue.Labels is left nil (callers MUST treat as empty).
// Opt-in performance flag for the bd list --skip-labels code path.
SkipLabels bool
// Performance escape hatches
SkipWisps bool // Q2: skip wisps table merge entirely (for callers that never return ephemeral results)
NoIDShrink bool // Q3: force Pattern A (full 47-col scan) even when Limit > 0
}
// SortPolicy determines how ready work is ordered