perf(label): count list-all labels from hydrated SearchIssues result (#5328)
bd label list-all fetched every issue via SearchIssues, which already hydrates Issue.Labels in bulk, then discarded that hydration and called store.GetLabels once per issue. In embedded Dolt mode each of those calls enters withConn, opens a short-lived OpenSQL connector, starts a transaction, runs a wisp-routing probe plus the label SELECT, and closes — 2N extra round trips for data already in memory. On a live 455-issue embedded database that turned a ~10s scan into a 33.75s command, and one run was killed at 104.31s with "context canceled while getting labels for mr-qji", the error string from exactly this loop. Count from issue.Labels instead. Output is unchanged: same unique labels, same counts, same sort order, same JSON shape. The helper takes a labelListAllSearcher interface exposing only SearchIssues, so the per-issue lookup cannot come back without widening that interface; a gate test asserts the method set. Benchmarked against a real embedded Dolt store: 200 issues, 5,960ms/op before vs 64.3ms/op after (92.6x); 20 issues, 478.7ms vs 27.3ms (17.5x). This is the same defect pattern that #1586 removed from the RPC handleList path; the proxied-server path was already bulk. Fixes #5325 Agent-Signature: claude-code-claude-opus-5-unknown-reasoning on behalf of mwotton
This commit is contained in:
parent
294af94be2
commit
160707b0bc
3 changed files with 276 additions and 14 deletions
|
|
@ -296,6 +296,29 @@ var labelListCmd = &cobra.Command{
|
|||
return runLabelList(rootCtx, args)
|
||||
},
|
||||
}
|
||||
|
||||
// labelListAllSearcher is the whole storage surface `bd label list-all`
|
||||
// needs. SearchIssues already hydrates Issue.Labels in bulk; omitting
|
||||
// GetLabels here keeps the per-issue lookup — one extra connection and
|
||||
// transaction per issue in embedded mode — unreachable (GH#5325).
|
||||
type labelListAllSearcher interface {
|
||||
SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error)
|
||||
}
|
||||
|
||||
func countLabelsAcrossIssues(ctx context.Context, s labelListAllSearcher) (map[string]int, error) {
|
||||
issues, err := s.SearchIssues(ctx, "", types.IssueFilter{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labelCounts := make(map[string]int)
|
||||
for _, issue := range issues {
|
||||
for _, label := range issue.Labels {
|
||||
labelCounts[label]++
|
||||
}
|
||||
}
|
||||
return labelCounts, nil
|
||||
}
|
||||
|
||||
var labelListAllCmd = &cobra.Command{
|
||||
Use: "list-all",
|
||||
Short: "List all unique labels in the database",
|
||||
|
|
@ -313,23 +336,10 @@ var labelListAllCmd = &cobra.Command{
|
|||
return runLabelListAllProxiedServer(rootCtx)
|
||||
}
|
||||
|
||||
ctx := rootCtx
|
||||
var issues []*types.Issue
|
||||
var err error
|
||||
issues, err = store.SearchIssues(ctx, "", types.IssueFilter{})
|
||||
labelCounts, err := countLabelsAcrossIssues(rootCtx, store)
|
||||
if err != nil {
|
||||
return HandleErrorRespectJSON("%v", err)
|
||||
}
|
||||
labelCounts := make(map[string]int)
|
||||
for _, issue := range issues {
|
||||
labels, err := store.GetLabels(ctx, issue.ID)
|
||||
if err != nil {
|
||||
return HandleErrorRespectJSON("getting labels for %s: %v", issue.ID, err)
|
||||
}
|
||||
for _, label := range labels {
|
||||
labelCounts[label]++
|
||||
}
|
||||
}
|
||||
type labelInfo struct {
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
|
|
|
|||
131
cmd/bd/label_list_all_test.go
Normal file
131
cmd/bd/label_list_all_test.go
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// searchOnlyStore records the SearchIssues calls countLabelsAcrossIssues
|
||||
// makes and returns issues whose Labels are already hydrated, the way
|
||||
// issueops.SearchIssuesInTx hydrates them for a real store.
|
||||
type searchOnlyStore struct {
|
||||
issues []*types.Issue
|
||||
err error
|
||||
|
||||
calls int
|
||||
filters []types.IssueFilter
|
||||
queries []string
|
||||
}
|
||||
|
||||
func (s *searchOnlyStore) SearchIssues(_ context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) {
|
||||
s.calls++
|
||||
s.queries = append(s.queries, query)
|
||||
s.filters = append(s.filters, filter)
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
return s.issues, nil
|
||||
}
|
||||
|
||||
func TestCountLabelsAcrossIssues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
issues []*types.Issue
|
||||
want map[string]int
|
||||
}{
|
||||
{
|
||||
name: "counts every hydrated label once per issue",
|
||||
issues: []*types.Issue{
|
||||
{ID: "be-1", Labels: []string{"backend", "urgent"}},
|
||||
{ID: "be-2", Labels: []string{"backend"}},
|
||||
{ID: "be-3", Labels: []string{"urgent", "docs"}},
|
||||
},
|
||||
want: map[string]int{"backend": 2, "urgent": 2, "docs": 1},
|
||||
},
|
||||
{
|
||||
name: "issues without labels contribute nothing",
|
||||
issues: []*types.Issue{{ID: "be-1"}, {ID: "be-2", Labels: []string{}}},
|
||||
want: map[string]int{},
|
||||
},
|
||||
{
|
||||
name: "no issues",
|
||||
issues: nil,
|
||||
want: map[string]int{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := &searchOnlyStore{issues: tt.issues}
|
||||
got, err := countLabelsAcrossIssues(context.Background(), store)
|
||||
if err != nil {
|
||||
t.Fatalf("countLabelsAcrossIssues: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("counts = %v, want %v", got, tt.want)
|
||||
}
|
||||
if store.calls != 1 {
|
||||
t.Errorf("SearchIssues called %d times, want exactly 1", store.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCountLabelsAcrossIssues_SearchFilter locks the search contract the
|
||||
// command depends on: an empty query and a zero IssueFilter, which means
|
||||
// full-database scope, bulk label hydration (SkipLabels stays false), and
|
||||
// the CONTRIBUTING.md MaxRows opt-out (this is not a --max-rows-wired
|
||||
// command, so the cap must never abort it).
|
||||
func TestCountLabelsAcrossIssues_SearchFilter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := &searchOnlyStore{issues: []*types.Issue{{ID: "be-1", Labels: []string{"x"}}}}
|
||||
if _, err := countLabelsAcrossIssues(context.Background(), store); err != nil {
|
||||
t.Fatalf("countLabelsAcrossIssues: %v", err)
|
||||
}
|
||||
if len(store.filters) != 1 {
|
||||
t.Fatalf("SearchIssues called %d times, want exactly 1", len(store.filters))
|
||||
}
|
||||
if store.queries[0] != "" {
|
||||
t.Errorf("query = %q, want empty (whole database)", store.queries[0])
|
||||
}
|
||||
if got := store.filters[0]; !reflect.DeepEqual(got, types.IssueFilter{}) {
|
||||
t.Errorf("filter = %+v, want the zero IssueFilter", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountLabelsAcrossIssues_SearchError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
want := errors.New("boom")
|
||||
store := &searchOnlyStore{err: want}
|
||||
if _, err := countLabelsAcrossIssues(context.Background(), store); !errors.Is(err, want) {
|
||||
t.Fatalf("err = %v, want %v", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLabelListAllSearcherExposesNoPerIssueLookup is the regression gate for
|
||||
// GH#5325. bd label list-all used to call store.GetLabels once per issue on
|
||||
// top of a SearchIssues result that already carried hydrated labels; in
|
||||
// embedded Dolt mode that is a fresh connector plus transaction per issue.
|
||||
// Keeping the command's storage surface to SearchIssues alone makes that
|
||||
// N+1 unreachable at compile time — widening this interface would silently
|
||||
// re-open it, so the method set is asserted directly.
|
||||
func TestLabelListAllSearcherExposesNoPerIssueLookup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
iface := reflect.TypeOf((*labelListAllSearcher)(nil)).Elem()
|
||||
for i := 0; i < iface.NumMethod(); i++ {
|
||||
if name := iface.Method(i).Name; name != "SearchIssues" {
|
||||
t.Errorf("labelListAllSearcher exposes %q; bd label list-all must read labels only from the bulk-hydrated SearchIssues result (GH#5325)", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
121
internal/storage/embeddeddolt/label_list_all_benchmark_test.go
Normal file
121
internal/storage/embeddeddolt/label_list_all_benchmark_test.go
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
//go:build cgo
|
||||
|
||||
package embeddeddolt_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage/embeddeddolt"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// BenchmarkLabelListAllLabelCounting measures the two ways `bd label list-all`
|
||||
// can total every label in the database against a real embedded Dolt store.
|
||||
//
|
||||
// - bulk_hydrated is what the command does now: one SearchIssues call, then
|
||||
// count from the Issue.Labels that issueops.SearchIssuesInTx already
|
||||
// hydrated in bulk.
|
||||
// - per_issue_getlabels is what it did before GH#5325: the same SearchIssues
|
||||
// call, then Store.GetLabels once per issue. In embedded mode each of
|
||||
// those opens a short-lived connector, starts a transaction, runs a wisp
|
||||
// probe plus the label SELECT, and closes.
|
||||
//
|
||||
// Workload: BEADS_BENCH_ISSUES issues (default 200), each carrying two labels
|
||||
// drawn from a small shared vocabulary, so both arms return identical counts.
|
||||
// Set BEADS_TEST_EMBEDDED_DOLT=1 to run.
|
||||
func BenchmarkLabelListAllLabelCounting(b *testing.B) {
|
||||
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
|
||||
b.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt benchmarks")
|
||||
}
|
||||
|
||||
issueCount := 200
|
||||
if raw := os.Getenv("BEADS_BENCH_ISSUES"); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil || parsed <= 0 {
|
||||
b.Fatalf("BEADS_BENCH_ISSUES=%q: want a positive integer", raw)
|
||||
}
|
||||
issueCount = parsed
|
||||
}
|
||||
|
||||
ctx := b.Context()
|
||||
beadsDir := filepath.Join(b.TempDir(), ".beads")
|
||||
store, err := embeddeddolt.Open(ctx, beadsDir, "bench", "main")
|
||||
if err != nil {
|
||||
b.Fatalf("open embedded Dolt store: %v", err)
|
||||
}
|
||||
b.Cleanup(func() {
|
||||
if err := store.Close(); err != nil {
|
||||
b.Errorf("close embedded Dolt store: %v", err)
|
||||
}
|
||||
})
|
||||
if err := store.SetConfig(ctx, "issue_prefix", "bench"); err != nil {
|
||||
b.Fatalf("set issue_prefix: %v", err)
|
||||
}
|
||||
|
||||
for i := range issueCount {
|
||||
issue := &types.Issue{
|
||||
ID: fmt.Sprintf("bench-%04d", i),
|
||||
Title: fmt.Sprintf("Bench issue %d", i),
|
||||
Status: types.StatusOpen,
|
||||
Priority: 2,
|
||||
IssueType: types.TypeTask,
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue, "bench"); err != nil {
|
||||
b.Fatalf("CreateIssue %s: %v", issue.ID, err)
|
||||
}
|
||||
for _, label := range []string{fmt.Sprintf("area-%d", i%7), fmt.Sprintf("tier-%d", i%3)} {
|
||||
if err := store.AddLabel(ctx, issue.ID, label, "bench"); err != nil {
|
||||
b.Fatalf("AddLabel %s/%s: %v", issue.ID, label, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := store.Commit(ctx, "bench seed"); err != nil {
|
||||
b.Fatalf("commit seed: %v", err)
|
||||
}
|
||||
|
||||
wantLabels := min(issueCount, 7) + min(issueCount, 3)
|
||||
|
||||
b.Run("bulk_hydrated", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
issues, err := store.SearchIssues(ctx, "", types.IssueFilter{})
|
||||
if err != nil {
|
||||
b.Fatalf("SearchIssues: %v", err)
|
||||
}
|
||||
counts := make(map[string]int)
|
||||
for _, issue := range issues {
|
||||
for _, label := range issue.Labels {
|
||||
counts[label]++
|
||||
}
|
||||
}
|
||||
if len(counts) != wantLabels {
|
||||
b.Fatalf("unique labels = %d, want %d", len(counts), wantLabels)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("per_issue_getlabels", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
issues, err := store.SearchIssues(ctx, "", types.IssueFilter{})
|
||||
if err != nil {
|
||||
b.Fatalf("SearchIssues: %v", err)
|
||||
}
|
||||
counts := make(map[string]int)
|
||||
for _, issue := range issues {
|
||||
labels, err := store.GetLabels(ctx, issue.ID)
|
||||
if err != nil {
|
||||
b.Fatalf("GetLabels %s: %v", issue.ID, err)
|
||||
}
|
||||
for _, label := range labels {
|
||||
counts[label]++
|
||||
}
|
||||
}
|
||||
if len(counts) != wantLabels {
|
||||
b.Fatalf("unique labels = %d, want %d", len(counts), wantLabels)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue