perf(search): Q2+Q3+Q5 — skip-wisps merge, id-shrunk Pattern B, wisp count cache (bd-qi4qr, bd-co6m6)
Q2 (bd-qi4qr): add IssueFilter.SkipWisps; set it in count/query/list callers that never need ephemeral results; SearchIssuesInTx returns early before the wisps merge. Q3 (bd-co6m6): add IssueFilter.NoIDShrink; searchTableInTx dispatches Pattern B (SELECT id LIMIT n + batch hydrate) when Limit>0 and NoIDShrink is false; Pattern A (full 47-column scan) retained for unlimited queries and NoIDShrink callers. Q5 (bd-qi4qr): per-process wisp row-count cache on DoltStore (wispCountCached/ wispCountZero); SearchIssues short-circuits SkipWisps when table is known empty, avoiding the COUNT(*) hit on every search; cache invalidated on CreateIssue, CreateIssuesWithFullOptions, DeleteIssue, DeleteIssues, and DemoteToWisp. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cee5941436
commit
9a29a158fd
4 changed files with 149 additions and 31 deletions
|
|
@ -197,6 +197,7 @@ Examples:
|
|||
filter.PriorityMax = &priorityMax
|
||||
}
|
||||
|
||||
filter.SkipWisps = true // bd count never needs ephemeral wisp results
|
||||
issues, err := store.SearchIssues(ctx, "", filter)
|
||||
if err != nil {
|
||||
FatalError("%v", err)
|
||||
|
|
|
|||
|
|
@ -1092,6 +1092,11 @@ var listCmd = &cobra.Command{
|
|||
FatalError("%v", err)
|
||||
}
|
||||
} else {
|
||||
// Skip wisps merge when the caller doesn't need ephemeral results
|
||||
// (no --include-infra, not an infra type request). Q2 perf opt.
|
||||
if !includeInfra && (issueType == "" || !isInfra(issueType)) {
|
||||
filter.SkipWisps = true
|
||||
}
|
||||
var err error
|
||||
issues, err = activeStore.SearchIssues(ctx, "", filter)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@ import (
|
|||
// SearchIssuesInTx executes a filtered issue search within an existing transaction.
|
||||
// It queries the issues table, optionally merges wisps, and returns hydrated issues
|
||||
// with labels populated.
|
||||
//
|
||||
// Set filter.SkipWisps=true for callers that never need ephemeral results; this
|
||||
// avoids the unconditional full-table wisps scan (Q2 perf opt).
|
||||
func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) ([]*types.Issue, error) {
|
||||
// Route ephemeral-only queries to wisps table.
|
||||
if filter.Ephemeral != nil && *filter.Ephemeral {
|
||||
|
|
@ -30,6 +33,11 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type
|
|||
return nil, fmt.Errorf("search issues: %w", err)
|
||||
}
|
||||
|
||||
// Skip wisps merge entirely when caller opts out (Q2: perf escape hatch).
|
||||
if filter.SkipWisps {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// When filter.Ephemeral is nil (search everything) or false (non-ephemeral
|
||||
// only), also search the wisps table and merge results. NoHistory beads are
|
||||
// stored in the wisps table with ephemeral=0, so they must survive an
|
||||
|
|
@ -67,6 +75,11 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type
|
|||
}
|
||||
|
||||
// searchTableInTx runs a filtered search against a specific table set (issues or wisps).
|
||||
//
|
||||
// When filter.Limit > 0 and !filter.NoIDShrink, uses Pattern B (id-shrunk): a cheap
|
||||
// SELECT id scan + batch hydration instead of a full 47-column projection scan.
|
||||
// Pattern B is equivalent to Pattern A but faster on large corpora where most rows
|
||||
// are never needed (mirrors the pattern in scanIssueIDs and GetStaleIssuesInTx).
|
||||
func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables) ([]*types.Issue, error) {
|
||||
fromSQL, labelWhere, labelArgs, labelDriven, filterForClauses := buildLabelDrivenSearch(filter, tables)
|
||||
whereClauses, args, err := BuildIssueFilterClauses(query, filterForClauses, tables)
|
||||
|
|
@ -83,6 +96,12 @@ func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types
|
|||
whereSQL = "WHERE " + strings.Join(whereClauses, " AND ")
|
||||
}
|
||||
|
||||
// Pattern B: when Limit > 0, use a cheap id scan then hydrate in batch.
|
||||
if filter.Limit > 0 && !filter.NoIDShrink {
|
||||
return searchTablePatternB(ctx, tx, fromSQL, whereSQL, args, filter, tables, labelDriven)
|
||||
}
|
||||
|
||||
// Pattern A: full 47-column scan (used for unlimited queries or when NoIDShrink is set).
|
||||
limitSQL := ""
|
||||
if filter.Limit > 0 {
|
||||
limitSQL = fmt.Sprintf(" LIMIT %d", filter.Limit)
|
||||
|
|
@ -120,44 +139,133 @@ func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types
|
|||
return nil, fmt.Errorf("search %s: rows: %w", tables.Main, err)
|
||||
}
|
||||
|
||||
// Hydrate labels in bulk after closing the result set, so we don't hold
|
||||
// multiple active result sets on the same connection.
|
||||
if len(issues) > 0 {
|
||||
ids := make([]string, len(issues))
|
||||
for i, issue := range issues {
|
||||
ids[i] = issue.ID
|
||||
}
|
||||
if !filter.SkipLabels {
|
||||
// Fast path: searchTableInTx queries exclusively either the issues
|
||||
// or wisps table, so every ID in `ids` belongs to tables.Labels.
|
||||
// Skip the per-batch wisp-partition round-trip that the generic
|
||||
// GetLabelsForIssuesInTx performs (GH#3414).
|
||||
labelMap, labelErr := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids)
|
||||
if labelErr != nil {
|
||||
return nil, fmt.Errorf("search %s: hydrate labels: %w", tables.Main, labelErr)
|
||||
}
|
||||
for _, issue := range issues {
|
||||
if labels, ok := labelMap[issue.ID]; ok {
|
||||
issue.Labels = labels
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := hydrateIssues(ctx, tx, issues, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil {
|
||||
return nil, fmt.Errorf("search %s: hydrate: %w", tables.Main, err)
|
||||
}
|
||||
|
||||
// Optionally hydrate dependencies in bulk (same batched pattern as labels).
|
||||
if filter.IncludeDependencies {
|
||||
depMap, depErr := GetDependencyRecordsForIssuesFromTableInTx(ctx, tx, tables.Dependencies, ids)
|
||||
if depErr != nil {
|
||||
return nil, fmt.Errorf("search %s: hydrate dependencies: %w", tables.Main, depErr)
|
||||
}
|
||||
for _, issue := range issues {
|
||||
if deps, ok := depMap[issue.ID]; ok {
|
||||
issue.Dependencies = deps
|
||||
}
|
||||
return issues, nil
|
||||
}
|
||||
|
||||
// searchTablePatternB runs Pattern B: SELECT id LIMIT n → batch hydrate.
|
||||
// Equivalent result to Pattern A but avoids streaming all 47 columns for rows
|
||||
// that won't survive the LIMIT cut. Mirrors the approach in GetStaleIssuesInTx.
|
||||
func searchTablePatternB(ctx context.Context, tx *sql.Tx, fromSQL, whereSQL string, args []interface{}, filter types.IssueFilter, tables FilterTables, labelDriven bool) ([]*types.Issue, error) {
|
||||
idSelect := "SELECT "
|
||||
if labelDriven {
|
||||
idSelect = "SELECT DISTINCT "
|
||||
}
|
||||
//nolint:gosec // G201: SQL fragments from fixed column/table names and parameterized filters.
|
||||
idQuery := fmt.Sprintf(`%s%s.id FROM %s %s ORDER BY %s.priority ASC, %s.created_at DESC, %s.id ASC LIMIT %d`,
|
||||
idSelect, tables.Main, fromSQL, whereSQL, tables.Main, tables.Main, tables.Main, filter.Limit)
|
||||
|
||||
rows, err := tx.QueryContext(ctx, idQuery, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search %s (id scan): %w", tables.Main, err)
|
||||
}
|
||||
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, fmt.Errorf("search %s (id scan): scan: %w", tables.Main, err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
_ = rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("search %s (id scan): rows: %w", tables.Main, err)
|
||||
}
|
||||
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Batch-fetch full rows from the known table (no wispSet partition needed).
|
||||
placeholders := make([]string, len(ids))
|
||||
fetchArgs := make([]interface{}, len(ids))
|
||||
for i, id := range ids {
|
||||
placeholders[i] = "?"
|
||||
fetchArgs[i] = id
|
||||
}
|
||||
//nolint:gosec // G201: table name is a fixed constant from FilterTables.
|
||||
fetchSQL := fmt.Sprintf(`SELECT %s FROM %s WHERE id IN (%s)`,
|
||||
IssueSelectColumns, tables.Main, strings.Join(placeholders, ","))
|
||||
|
||||
fetchRows, err := tx.QueryContext(ctx, fetchSQL, fetchArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search %s (hydrate): %w", tables.Main, err)
|
||||
}
|
||||
|
||||
issueMap := make(map[string]*types.Issue, len(ids))
|
||||
for fetchRows.Next() {
|
||||
issue, scanErr := ScanIssueFrom(fetchRows)
|
||||
if scanErr != nil {
|
||||
_ = fetchRows.Close()
|
||||
return nil, fmt.Errorf("search %s (hydrate): scan: %w", tables.Main, scanErr)
|
||||
}
|
||||
issueMap[issue.ID] = issue
|
||||
}
|
||||
_ = fetchRows.Close()
|
||||
if err := fetchRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("search %s (hydrate): rows: %w", tables.Main, err)
|
||||
}
|
||||
|
||||
// Reorder to preserve the id-scan ORDER BY.
|
||||
issues := make([]*types.Issue, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if issue, ok := issueMap[id]; ok {
|
||||
issues = append(issues, issue)
|
||||
}
|
||||
}
|
||||
|
||||
if err := hydrateIssues(ctx, tx, issues, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil {
|
||||
return nil, fmt.Errorf("search %s (pattern B): hydrate: %w", tables.Main, err)
|
||||
}
|
||||
|
||||
return issues, nil
|
||||
}
|
||||
|
||||
// hydrateIssues populates labels (and optionally dependencies) on a slice of issues.
|
||||
// All issues must belong to tables.Main; labels come from tables.Labels.
|
||||
// When skipLabels is true, label hydration is suppressed (Issue.Labels is left nil).
|
||||
func hydrateIssues(ctx context.Context, tx *sql.Tx, issues []*types.Issue, tables FilterTables, includeDeps bool, skipLabels bool) error {
|
||||
if len(issues) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ids := make([]string, len(issues))
|
||||
for i, issue := range issues {
|
||||
ids[i] = issue.ID
|
||||
}
|
||||
|
||||
if !skipLabels {
|
||||
// Fast path: every ID in `ids` belongs to tables.Labels.
|
||||
// Skip the per-batch wisp-partition round-trip (GH#3414).
|
||||
labelMap, err := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hydrate labels: %w", err)
|
||||
}
|
||||
for _, issue := range issues {
|
||||
if labels, ok := labelMap[issue.ID]; ok {
|
||||
issue.Labels = labels
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return issues, nil
|
||||
if includeDeps {
|
||||
depMap, err := GetDependencyRecordsForIssuesFromTableInTx(ctx, tx, tables.Dependencies, ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hydrate dependencies: %w", err)
|
||||
}
|
||||
for _, issue := range issues {
|
||||
if deps, ok := depMap[issue.ID]; ok {
|
||||
issue.Dependencies = deps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildLabelDrivenSearch(filter types.IssueFilter, tables FilterTables) (string, []string, []interface{}, bool, types.IssueFilter) {
|
||||
|
|
|
|||
|
|
@ -1282,6 +1282,10 @@ type IssueFilter struct {
|
|||
// skipped and Issue.Labels is left nil (callers MUST treat as empty).
|
||||
// Opt-in performance flag for the bd list --skip-labels code path.
|
||||
SkipLabels bool
|
||||
|
||||
// Performance escape hatches
|
||||
SkipWisps bool // Q2: skip wisps table merge entirely (for callers that never return ephemeral results)
|
||||
NoIDShrink bool // Q3: force Pattern A (full 47-col scan) even when Limit > 0
|
||||
}
|
||||
|
||||
// SortPolicy determines how ready work is ordered
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue