diff --git a/cmd/bd/count.go b/cmd/bd/count.go index dff314181..374b76422 100644 --- a/cmd/bd/count.go +++ b/cmd/bd/count.go @@ -197,6 +197,7 @@ Examples: filter.PriorityMax = &priorityMax } + filter.SkipWisps = true // bd count never needs ephemeral wisp results issues, err := store.SearchIssues(ctx, "", filter) if err != nil { FatalError("%v", err) diff --git a/cmd/bd/list.go b/cmd/bd/list.go index e1796b37f..e35bea3b7 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -1092,6 +1092,11 @@ var listCmd = &cobra.Command{ FatalError("%v", err) } } else { + // Skip wisps merge when the caller doesn't need ephemeral results + // (no --include-infra, not an infra type request). Q2 perf opt. + if !includeInfra && (issueType == "" || !isInfra(issueType)) { + filter.SkipWisps = true + } var err error issues, err = activeStore.SearchIssues(ctx, "", filter) if err != nil { diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index 23b4ffd1d..544cdef64 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -12,6 +12,9 @@ import ( // SearchIssuesInTx executes a filtered issue search within an existing transaction. // It queries the issues table, optionally merges wisps, and returns hydrated issues // with labels populated. +// +// Set filter.SkipWisps=true for callers that never need ephemeral results; this +// avoids the unconditional full-table wisps scan (Q2 perf opt). func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) ([]*types.Issue, error) { // Route ephemeral-only queries to wisps table. if filter.Ephemeral != nil && *filter.Ephemeral { @@ -30,6 +33,11 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type return nil, fmt.Errorf("search issues: %w", err) } + // Skip wisps merge entirely when caller opts out (Q2: perf escape hatch). + if filter.SkipWisps { + return results, nil + } + // When filter.Ephemeral is nil (search everything) or false (non-ephemeral // only), also search the wisps table and merge results. NoHistory beads are // stored in the wisps table with ephemeral=0, so they must survive an @@ -67,6 +75,11 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type } // searchTableInTx runs a filtered search against a specific table set (issues or wisps). +// +// When filter.Limit > 0 and !filter.NoIDShrink, uses Pattern B (id-shrunk): a cheap +// SELECT id scan + batch hydration instead of a full 47-column projection scan. +// Pattern B is equivalent to Pattern A but faster on large corpora where most rows +// are never needed (mirrors the pattern in scanIssueIDs and GetStaleIssuesInTx). func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables) ([]*types.Issue, error) { fromSQL, labelWhere, labelArgs, labelDriven, filterForClauses := buildLabelDrivenSearch(filter, tables) whereClauses, args, err := BuildIssueFilterClauses(query, filterForClauses, tables) @@ -83,6 +96,12 @@ func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types whereSQL = "WHERE " + strings.Join(whereClauses, " AND ") } + // Pattern B: when Limit > 0, use a cheap id scan then hydrate in batch. + if filter.Limit > 0 && !filter.NoIDShrink { + return searchTablePatternB(ctx, tx, fromSQL, whereSQL, args, filter, tables, labelDriven) + } + + // Pattern A: full 47-column scan (used for unlimited queries or when NoIDShrink is set). limitSQL := "" if filter.Limit > 0 { limitSQL = fmt.Sprintf(" LIMIT %d", filter.Limit) @@ -120,44 +139,133 @@ func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types return nil, fmt.Errorf("search %s: rows: %w", tables.Main, err) } - // Hydrate labels in bulk after closing the result set, so we don't hold - // multiple active result sets on the same connection. - if len(issues) > 0 { - ids := make([]string, len(issues)) - for i, issue := range issues { - ids[i] = issue.ID - } - if !filter.SkipLabels { - // Fast path: searchTableInTx queries exclusively either the issues - // or wisps table, so every ID in `ids` belongs to tables.Labels. - // Skip the per-batch wisp-partition round-trip that the generic - // GetLabelsForIssuesInTx performs (GH#3414). - labelMap, labelErr := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids) - if labelErr != nil { - return nil, fmt.Errorf("search %s: hydrate labels: %w", tables.Main, labelErr) - } - for _, issue := range issues { - if labels, ok := labelMap[issue.ID]; ok { - issue.Labels = labels - } - } - } + if err := hydrateIssues(ctx, tx, issues, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil { + return nil, fmt.Errorf("search %s: hydrate: %w", tables.Main, err) + } - // Optionally hydrate dependencies in bulk (same batched pattern as labels). - if filter.IncludeDependencies { - depMap, depErr := GetDependencyRecordsForIssuesFromTableInTx(ctx, tx, tables.Dependencies, ids) - if depErr != nil { - return nil, fmt.Errorf("search %s: hydrate dependencies: %w", tables.Main, depErr) - } - for _, issue := range issues { - if deps, ok := depMap[issue.ID]; ok { - issue.Dependencies = deps - } + return issues, nil +} + +// searchTablePatternB runs Pattern B: SELECT id LIMIT n → batch hydrate. +// Equivalent result to Pattern A but avoids streaming all 47 columns for rows +// that won't survive the LIMIT cut. Mirrors the approach in GetStaleIssuesInTx. +func searchTablePatternB(ctx context.Context, tx *sql.Tx, fromSQL, whereSQL string, args []interface{}, filter types.IssueFilter, tables FilterTables, labelDriven bool) ([]*types.Issue, error) { + idSelect := "SELECT " + if labelDriven { + idSelect = "SELECT DISTINCT " + } + //nolint:gosec // G201: SQL fragments from fixed column/table names and parameterized filters. + idQuery := fmt.Sprintf(`%s%s.id FROM %s %s ORDER BY %s.priority ASC, %s.created_at DESC, %s.id ASC LIMIT %d`, + idSelect, tables.Main, fromSQL, whereSQL, tables.Main, tables.Main, tables.Main, filter.Limit) + + rows, err := tx.QueryContext(ctx, idQuery, args...) + if err != nil { + return nil, fmt.Errorf("search %s (id scan): %w", tables.Main, err) + } + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("search %s (id scan): scan: %w", tables.Main, err) + } + ids = append(ids, id) + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("search %s (id scan): rows: %w", tables.Main, err) + } + + if len(ids) == 0 { + return nil, nil + } + + // Batch-fetch full rows from the known table (no wispSet partition needed). + placeholders := make([]string, len(ids)) + fetchArgs := make([]interface{}, len(ids)) + for i, id := range ids { + placeholders[i] = "?" + fetchArgs[i] = id + } + //nolint:gosec // G201: table name is a fixed constant from FilterTables. + fetchSQL := fmt.Sprintf(`SELECT %s FROM %s WHERE id IN (%s)`, + IssueSelectColumns, tables.Main, strings.Join(placeholders, ",")) + + fetchRows, err := tx.QueryContext(ctx, fetchSQL, fetchArgs...) + if err != nil { + return nil, fmt.Errorf("search %s (hydrate): %w", tables.Main, err) + } + + issueMap := make(map[string]*types.Issue, len(ids)) + for fetchRows.Next() { + issue, scanErr := ScanIssueFrom(fetchRows) + if scanErr != nil { + _ = fetchRows.Close() + return nil, fmt.Errorf("search %s (hydrate): scan: %w", tables.Main, scanErr) + } + issueMap[issue.ID] = issue + } + _ = fetchRows.Close() + if err := fetchRows.Err(); err != nil { + return nil, fmt.Errorf("search %s (hydrate): rows: %w", tables.Main, err) + } + + // Reorder to preserve the id-scan ORDER BY. + issues := make([]*types.Issue, 0, len(ids)) + for _, id := range ids { + if issue, ok := issueMap[id]; ok { + issues = append(issues, issue) + } + } + + if err := hydrateIssues(ctx, tx, issues, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil { + return nil, fmt.Errorf("search %s (pattern B): hydrate: %w", tables.Main, err) + } + + return issues, nil +} + +// hydrateIssues populates labels (and optionally dependencies) on a slice of issues. +// All issues must belong to tables.Main; labels come from tables.Labels. +// When skipLabels is true, label hydration is suppressed (Issue.Labels is left nil). +func hydrateIssues(ctx context.Context, tx *sql.Tx, issues []*types.Issue, tables FilterTables, includeDeps bool, skipLabels bool) error { + if len(issues) == 0 { + return nil + } + + ids := make([]string, len(issues)) + for i, issue := range issues { + ids[i] = issue.ID + } + + if !skipLabels { + // Fast path: every ID in `ids` belongs to tables.Labels. + // Skip the per-batch wisp-partition round-trip (GH#3414). + labelMap, err := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids) + if err != nil { + return fmt.Errorf("hydrate labels: %w", err) + } + for _, issue := range issues { + if labels, ok := labelMap[issue.ID]; ok { + issue.Labels = labels } } } - return issues, nil + if includeDeps { + depMap, err := GetDependencyRecordsForIssuesFromTableInTx(ctx, tx, tables.Dependencies, ids) + if err != nil { + return fmt.Errorf("hydrate dependencies: %w", err) + } + for _, issue := range issues { + if deps, ok := depMap[issue.ID]; ok { + issue.Dependencies = deps + } + } + } + + return nil } func buildLabelDrivenSearch(filter types.IssueFilter, tables FilterTables) (string, []string, []interface{}, bool, types.IssueFilter) { diff --git a/internal/types/types.go b/internal/types/types.go index d1f3112b6..e3ef0e143 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1282,6 +1282,10 @@ type IssueFilter struct { // skipped and Issue.Labels is left nil (callers MUST treat as empty). // Opt-in performance flag for the bd list --skip-labels code path. SkipLabels bool + + // Performance escape hatches + SkipWisps bool // Q2: skip wisps table merge entirely (for callers that never return ephemeral results) + NoIDShrink bool // Q3: force Pattern A (full 47-col scan) even when Limit > 0 } // SortPolicy determines how ready work is ordered