diff --git a/cmd/bd/export_proxied_integration_test.go b/cmd/bd/export_proxied_integration_test.go index 74296bf3f..cf695e1ec 100644 --- a/cmd/bd/export_proxied_integration_test.go +++ b/cmd/bd/export_proxied_integration_test.go @@ -107,6 +107,15 @@ func seedProxiedExportFixture(t *testing.T, bd string, p proxiedProject) exportP // Ephemeral wisp: excluded from default export, included with --all. fx.wisp = bdProxiedCreateSilent(t, bd, p.dir, "Ephemeral heartbeat", "--ephemeral", "--wisp-type", "heartbeat") + // Relations on the wisp: the --all export must carry them, which is what + // pins the UOW plane partition (only wisp ids reach the wisp readers, + // wy-237yfi) against the classic-mode oracle. + if out, err := bdProxiedRun(t, bd, p.dir, "label", "add", fx.wisp, "wisp-label"); err != nil { + t.Fatalf("label add on wisp: %v\n%s", err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "comment", fx.wisp, "wisp comment"); err != nil { + t.Fatalf("comment on wisp: %v\n%s", err, out) + } // Infra-typed bead (message): auto-routed to the ephemeral plane and an // infra type, so default export must exclude it and --all include it. @@ -409,6 +418,21 @@ func TestProxiedServerExport(t *testing.T) { if len(keys) != 2 || keys[0] != "aardvark" || keys[1] != "zebra" { t.Errorf("--all export memories = %v, want [aardvark zebra] (sorted)", keys) } + // The wisp's own relations ride along: a misclassified plane would + // drop them silently while the id above still passes. + for _, rec := range records { + if rec["id"] != fx.wisp { + continue + } + labels, _ := rec["labels"].([]any) + if len(labels) != 1 || labels[0] != "wisp-label" { + t.Errorf("--all export wisp %s labels = %v, want [wisp-label]", fx.wisp, rec["labels"]) + } + comments, _ := rec["comments"].([]any) + if len(comments) != 1 { + t.Errorf("--all export wisp %s comments = %v, want one", fx.wisp, rec["comments"]) + } + } }) t.Run("include_memories_after_issues_sorted", func(t *testing.T) { diff --git a/cmd/bd/export_source.go b/cmd/bd/export_source.go index 6eb0c08b9..df4e4acfd 100644 --- a/cmd/bd/export_source.go +++ b/cmd/bd/export_source.go @@ -207,10 +207,14 @@ func (s *uowExportSource) GetAllConfig(ctx context.Context) (map[string]string, // by wisps-table membership (issueops.PartitionWispIDsInTx). Row flags are // NOT a substitute for that partition — a promoted no-history wisp is a // durable issues-table row that (as wild data from the pre-bd-r9uce -// promote) still carries NoHistory=true — so both -// planes are queried with the FULL ID set and merged. The planes are -// ID-disjoint (GH#4455 cross-table guard), so at most one plane returns -// rows for any id and the merge cannot collide. +// promote) still carries NoHistory=true — so membership is read from the +// wisps table itself (WispPlaneIDs), never inferred. The durable plane is +// then queried with the FULL ID set and the wisp plane only with that +// membership subset (wispReaderIDs), and the two are merged. Unreadable +// membership fails OPEN to the full id list, so a nil set is never +// mistaken for "no wisps" (wy-237yfi). The planes are ID-disjoint +// (GH#4455 cross-table guard), so at most one plane returns rows for any +// id and the merge cannot collide. // - Dependency records: domain GetIssueDependencyRecords already partitions // internally (same issueops helper), so one call covers both planes. // - Dependency counts: classic GetDependencyCountsInTx queries BOTH dep @@ -219,7 +223,12 @@ func (s *uowExportSource) GetAllConfig(ctx context.Context) (map[string]string, // queried for the full ID set and summed here too. // // Wisp-plane reads tolerate a rig without the wisp tables (IsTableNotExist), -// mirroring the dependency-counts leg. +// mirroring the dependency-counts leg. The three wisp-relation readers below +// keep that blanket check: each is a single-table read, so blanket and +// table-specific are the same test there. The membership probe is the +// exception — WispPlaneIDs renders FROM wisps LEFT JOIN leases, so its +// tolerance is pinned to a missing `wisps` table and a broken database +// surfaces as an error instead of an empty wisp plane. func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*types.Issue) (exportRelations, error) { rel := exportRelations{ labels: map[string][]string{}, @@ -259,7 +268,24 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ } mergeExportMap(rel.commentCounts, counts) - wispLabels, err := s.uw.LabelUseCase().GetLabelsForWisps(ctx, allIDs) //nolint:forbidigo // bulk relation load; see GetLabelsForIssues above + // The wisp_labels / wisp_comments rows of a wisp are keyed by an issue_id + // that lives in the WISPS table (promotion moves them to the durable + // tables, issueops.PromoteFromEphemeralInTx), so the three wisp readers + // below only need the ids that are wisps right now. Handing them the + // whole workspace was pure waste — on a 19k-issue rig with no wisps it + // was three 19k-placeholder statements answering nothing (wy-237yfi). + // CountsByWispIDs is NOT narrowed: its inbound leg counts wisp -> durable + // edges under the durable target's id. + // A nil set means membership could NOT be read (the wisps table is + // absent, the tolerated case) — never "no wisps": fail open to the full + // id list so the readers below keep their own table-not-exist tolerance. + wispSet, err := s.WispPlaneIDs(ctx, allIDs) + if err != nil { + return exportRelations{}, err + } + wispIDs := wispReaderIDs(allIDs, wispSet) + + wispLabels, err := s.uw.LabelUseCase().GetLabelsForWisps(ctx, wispIDs) //nolint:forbidigo // bulk relation load; see GetLabelsForIssues above if err != nil { if !dberrors.IsTableNotExist(err) { return exportRelations{}, fmt.Errorf("load wisp labels: %w", err) @@ -267,7 +293,7 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ } else { mergeExportMap(rel.labels, wispLabels) } - wispComments, err := s.uw.CommentUseCase().GetCommentsForWisps(ctx, allIDs) + wispComments, err := s.uw.CommentUseCase().GetCommentsForWisps(ctx, wispIDs) if err != nil { if !dberrors.IsTableNotExist(err) { return exportRelations{}, fmt.Errorf("load wisp comments: %w", err) @@ -275,7 +301,7 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ } else { mergeExportMap(rel.comments, wispComments) } - wispCounts, err := s.uw.CommentUseCase().GetWispCommentCounts(ctx, allIDs) + wispCounts, err := s.uw.CommentUseCase().GetWispCommentCounts(ctx, wispIDs) if err != nil { if !dberrors.IsTableNotExist(err) { return exportRelations{}, fmt.Errorf("load wisp comment counts: %w", err) @@ -319,17 +345,47 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ return rel, nil } +// wispPlaneSubset returns, in input order, the ids that wispSet marks as +// wisp-plane; none marked yields nil, and the bulk readers answer nil with no +// statement at all. The caller decides what a nil SET means (see +// LoadExportRelations: unreadable membership fails open, never empty). +func wispPlaneSubset(ids []string, wispSet map[string]bool) []string { + var out []string + for _, id := range ids { + if wispSet[id] { + out = append(out, id) + } + } + return out +} + +// wispReaderIDs is the caller-side policy wispPlaneSubset defers: a nil set +// means membership was UNREADABLE (WispPlaneIDs tolerated a missing wisps +// table), so every id goes to the wisp readers and they keep their own +// table-not-exist tolerance; a non-nil set — even an empty one — is the +// answer, and only the ids in it go. +func wispReaderIDs(ids []string, wispSet map[string]bool) []string { + if wispSet == nil { + return ids + } + return wispPlaneSubset(ids, wispSet) +} + // WispPlaneIDs classifies by wisps-table membership through the plane-pinned // GetWispsByIDs read (row flags are NOT a substitute — see the exportSource -// interface comment). A rig without the wisp tables has no wisp-plane rows, -// mirroring the tolerance of the wisp-plane legs in LoadExportRelations. +// interface comment). A rig without the wisps table has no wisp-plane rows +// (nil set), mirroring the tolerance of the wisp-plane legs in +// LoadExportRelations. Only the WISPS table is optional: the read joins +// leases, and a blanket table-not-exist check would report a broken +// database as an empty wisp plane (the issue_search.go missingOptionalWispTable +// lesson), so the tolerance is pinned to that one table name. func (s *uowExportSource) WispPlaneIDs(ctx context.Context, ids []string) (map[string]bool, error) { if len(ids) == 0 { return nil, nil } wisps, err := s.uw.IssueUseCase().GetWispsByIDs(ctx, ids) if err != nil { - if dberrors.IsTableNotExist(err) { + if dberrors.IsMissingTable(err, "wisps") { return nil, nil } return nil, fmt.Errorf("wisp plane membership: %w", err) diff --git a/cmd/bd/export_source_wisp_subset_test.go b/cmd/bd/export_source_wisp_subset_test.go new file mode 100644 index 000000000..042d7a064 --- /dev/null +++ b/cmd/bd/export_source_wisp_subset_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "reflect" + "testing" +) + +// wispPlaneSubset feeds the wisp-plane relation readers in +// uowExportSource.LoadExportRelations: only ids that live in the wisps table +// go to them, in input order; a set that marks none sends nothing, so those +// readers issue no statement at all (wy-237yfi). +func TestWispPlaneSubset(t *testing.T) { + ids := []string{"a", "b", "c", "d"} + + if got := wispPlaneSubset(ids, map[string]bool{}); got != nil { + t.Fatalf("empty set: want nil, got %v", got) + } + got := wispPlaneSubset(ids, map[string]bool{"d": true, "b": true, "zz": true, "a": false}) + if !reflect.DeepEqual(got, []string{"b", "d"}) { + t.Fatalf("want [b d] in input order, got %v", got) + } + if got := wispPlaneSubset(nil, map[string]bool{"a": true}); got != nil { + t.Fatalf("no ids: want nil, got %v", got) + } +} + +// wispReaderIDs is the policy on top: a nil set is "membership unreadable" +// and must FAIL OPEN to every id — narrowing on it would silently export +// every wisp with no labels and no comments — while a non-nil empty set is a +// real answer (no wisps) and sends nothing. +func TestWispReaderIDs(t *testing.T) { + ids := []string{"a", "b", "c"} + + if got := wispReaderIDs(ids, nil); !reflect.DeepEqual(got, ids) { + t.Fatalf("nil set must fail open to all ids, got %v", got) + } + if got := wispReaderIDs(ids, map[string]bool{}); got != nil { + t.Fatalf("empty non-nil set means no wisps: want nil, got %v", got) + } + if got := wispReaderIDs(ids, map[string]bool{"c": true}); !reflect.DeepEqual(got, []string{"c"}) { + t.Fatalf("want [c], got %v", got) + } +} diff --git a/internal/storage/domain/db/batching.go b/internal/storage/domain/db/batching.go new file mode 100644 index 000000000..c48e4b6bd --- /dev/null +++ b/internal/storage/domain/db/batching.go @@ -0,0 +1,26 @@ +package db + +// forEachIDBatch calls fn once per queryBatchSize-sized slice of ids, in +// order, stopping at the first error. The bulk readers use it to keep every +// IN (...) list bounded: Dolt's cost per IN-list placeholder is super-linear, +// so one statement over N ids is far slower than ceil(N/queryBatchSize) +// statements over queryBatchSize each (measured on a 19k-issue rig: the same +// labels rows took 31s as one statement and 0.8s as 98 batched ones). +// +// A given id lands in exactly one batch, so per-id result groups — and the +// ORDER BY within each group — survive being merged across batches. +// +// Read paths only: the DeleteAllForIDs loops keep their own deleteBatchSize +// and are deliberately left open-coded. +func forEachIDBatch(ids []string, fn func(batch []string) error) error { + for start := 0; start < len(ids); start += queryBatchSize { + end := start + queryBatchSize + if end > len(ids) { + end = len(ids) + } + if err := fn(ids[start:end]); err != nil { + return err + } + } + return nil +} diff --git a/internal/storage/domain/db/batching_test.go b/internal/storage/domain/db/batching_test.go new file mode 100644 index 000000000..315d02634 --- /dev/null +++ b/internal/storage/domain/db/batching_test.go @@ -0,0 +1,394 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "strings" + "sync" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// recordingRunner delegates to a real Runner and records every statement it +// is asked to run, so a test can assert on how many round trips a bulk +// reader made and how many placeholders each one carried. +type recordingRunner struct { + Runner + mu sync.Mutex + calls []recordedCall +} + +type recordedCall struct { + query string + nargs int +} + +func (r *recordingRunner) record(query string, args []any) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls = append(r.calls, recordedCall{query: query, nargs: len(args)}) +} + +func (r *recordingRunner) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + r.record(query, args) + return r.Runner.QueryContext(ctx, query, args...) +} + +func (r *recordingRunner) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + r.record(query, args) + return r.Runner.QueryRowContext(ctx, query, args...) +} + +func (r *recordingRunner) reset() []recordedCall { + r.mu.Lock() + defer r.mu.Unlock() + out := r.calls + r.calls = nil + return out +} + +// TestBulkReadersBatch pins the fence wy-237yfi built: the bulk relation +// readers (comment, label, dependency) and GetByIDs send at most +// queryBatchSize ids per IN list, in ceil(N/queryBatchSize) statements per +// query leg, and merge the batches into one result that is indistinguishable +// from a single statement over all N ids. Dolt is super-linear in IN-list size (a 19k-id list took 24-60s +// per statement on the constellation's rig; 200-id batches take ~5ms each), +// so the placeholder cap is the property, not the statement count. +func (s *testSuite) TestBulkReadersBatch() { + const n = queryBatchSize*2 + 50 // three batches: 200, 200, 50 + ids := make([]string, n) + for i := range ids { + ids[i] = fmt.Sprintf("bd-batch-%03d", i) + } + // Relations sit on the ids at each batch boundary so a merge that + // dropped or duplicated a batch would show up in the result. + seeded := []int{0, queryBatchSize - 1, queryBatchSize, n - 1} + for _, i := range seeded { + s.seedIssueRow(ids[i]) + } + wantBatches := (n + queryBatchSize - 1) / queryBatchSize + + rec := &recordingRunner{Runner: s.Runner()} + comments := NewCommentSQLRepository(rec) + labels := NewLabelSQLRepository(rec) + deps := NewDependencySQLRepository(rec) + + base := time.Date(2026, time.September, 4, 0, 0, 0, 0, time.UTC) + for k, i := range seeded { + for j := 0; j <= k; j++ { + s.seedComment(ids[i], "tester", fmt.Sprintf("c%d", j), base.Add(time.Duration(j)*time.Second)) + } + s.Require().NoError(labels.Insert(s.Ctx(), ids[i], fmt.Sprintf("l%d", k), "tester", domain.LabelOpts{})) + } + // ids[0] blocks ids[n-1] (crosses batches 1 and 3); ids[200] blocks + // ids[199] (crosses batches 2 and 1). + s.Require().NoError(deps.Insert(s.Ctx(), newDep(ids[n-1], ids[0], types.DepBlocks), "tester", domain.DepInsertOpts{})) + s.Require().NoError(deps.Insert(s.Ctx(), newDep(ids[queryBatchSize-1], ids[queryBatchSize], types.DepBlocks), "tester", domain.DepInsertOpts{})) + rec.reset() + + // extra is the number of non-id placeholders a leg carries (a type + // filter); the id list itself is what the cap is about. + assertBatchedExtra := func(name string, legs, extra int) { + calls := rec.reset() + s.Require().Len(calls, wantBatches*legs, "%s: statements issued", name) + for _, c := range calls { + s.LessOrEqual(c.nargs, queryBatchSize+extra, "%s: bound args in one statement", name) + s.LessOrEqual(strings.Count(c.query, "?"), queryBatchSize+extra, "%s: placeholders in one statement", name) + s.Contains(c.query, " IN (", "%s: every leg is an IN-list read", name) + } + } + assertBatched := func(name string, legs int) { assertBatchedExtra(name, legs, 0) } + + // countMatching reports how many recorded statements contain sub — the + // readers below issue more than one KIND of statement per call (two + // dependency legs, then a status lookup per plane), so their legs are + // counted by shape rather than by a flat statement total. + countMatching := func(calls []recordedCall, sub string) int { + n := 0 + for _, c := range calls { + if strings.Contains(c.query, sub) { + n++ + } + } + return n + } + // assertUnderCap is the half of the theorem that holds for EVERY leg: no + // statement carries more than queryBatchSize id placeholders. + assertUnderCap := func(name string, calls []recordedCall) { + for _, c := range calls { + s.LessOrEqual(c.nargs, queryBatchSize, "%s: bound args in one statement", name) + s.LessOrEqual(strings.Count(c.query, "?"), queryBatchSize, "%s: placeholders in one statement", name) + } + } + + s.Run("CommentCounts", func() { + out, err := comments.CountsByIssueIDs(s.Ctx(), ids, domain.CommentOpts{}) + s.Require().NoError(err) + assertBatched("CountsByIssueIDs", 1) + s.Len(out, len(seeded)) + for k, i := range seeded { + s.Equal(k+1, out[ids[i]], "count for %s", ids[i]) + } + }) + + s.Run("CommentList", func() { + out, err := comments.ListByIssueIDs(s.Ctx(), ids, domain.CommentOpts{}) + s.Require().NoError(err) + assertBatched("ListByIssueIDs", 1) + s.Len(out, len(seeded)) + for k, i := range seeded { + got := out[ids[i]] + s.Require().Len(got, k+1, "comments for %s", ids[i]) + for j, c := range got { + s.Equal(fmt.Sprintf("c%d", j), c.Text, "order preserved within %s", ids[i]) + } + } + }) + + s.Run("Labels", func() { + out, err := labels.ListByIssueIDs(s.Ctx(), ids, domain.LabelOpts{}) + s.Require().NoError(err) + assertBatched("Labels.ListByIssueIDs", 1) + s.Len(out, len(seeded)) + for k, i := range seeded { + s.Equal([]string{fmt.Sprintf("l%d", k)}, out[ids[i]]) + } + }) + + s.Run("DependencyList", func() { + out, err := deps.ListByIssueIDs(s.Ctx(), ids, domain.DepListOpts{Direction: domain.DepDirectionBoth}) + s.Require().NoError(err) + assertBatched("Deps.ListByIssueIDs", 2) + s.Len(out.Outgoing, 2) + s.Len(out.Incoming, 2) + s.Require().Len(out.Outgoing[ids[n-1]], 1) + s.Equal(ids[0], out.Outgoing[ids[n-1]][0].DependsOnID) + s.Require().Len(out.Incoming[ids[0]], 1) + s.Equal(ids[n-1], out.Incoming[ids[0]][0].IssueID) + s.Require().Len(out.Outgoing[ids[queryBatchSize-1]], 1) + s.Require().Len(out.Incoming[ids[queryBatchSize]], 1) + }) + + s.Run("DependencyCounts", func() { + out, err := deps.CountsByIssueIDs(s.Ctx(), ids, domain.DepCountsOpts{}) + s.Require().NoError(err) + assertBatched("Deps.CountsByIssueIDs", 2) + s.Len(out, n, "every requested id is present, zero or not") + s.Equal(&types.DependencyCounts{DependencyCount: 1}, out[ids[n-1]]) + s.Equal(&types.DependencyCounts{DependentCount: 1}, out[ids[0]]) + s.Equal(&types.DependencyCounts{DependencyCount: 1}, out[ids[queryBatchSize-1]]) + s.Equal(&types.DependencyCounts{DependentCount: 1}, out[ids[queryBatchSize]]) + s.Equal(&types.DependencyCounts{}, out[ids[1]]) + }) + + s.Run("DependencyListTypeFilter", func() { + // typeArgs ride along after the id args in EVERY batch, so the + // filter must hold across all three and the placeholder cap grows + // by exactly the filter's width. + s.Require().NoError(deps.Insert(s.Ctx(), newDep(ids[0], ids[queryBatchSize], types.DepRelated), "tester", domain.DepInsertOpts{})) + rec.reset() + out, err := deps.ListByIssueIDs(s.Ctx(), ids, domain.DepListOpts{ + Direction: domain.DepDirectionBoth, + Types: []types.DependencyType{types.DepBlocks}, + }) + s.Require().NoError(err) + assertBatchedExtra("Deps.ListByIssueIDs(Types)", 2, 1) + s.Require().Len(out.Outgoing[ids[0]], 0, "the related edge is filtered out in batch 1") + s.Require().Len(out.Incoming[ids[queryBatchSize]], 1, "only the blocks edge survives in batch 2") + s.Equal(types.DepBlocks, out.Incoming[ids[queryBatchSize]][0].Type) + s.Require().Len(out.Outgoing[ids[n-1]], 1) + }) + + s.Run("WispPlane", func() { + // One wisp in the middle batch; every reader routed to the wisp + // tables must batch the same way and find it. + wisp := ids[queryBatchSize+50] + s.seedWispRow(wisp) + s.seedWispComment(wisp, "tester", "w0", base) + s.seedWispComment(wisp, "tester", "w1", base.Add(time.Second)) + s.Require().NoError(labels.Insert(s.Ctx(), wisp, "wl", "tester", domain.LabelOpts{UseWispsTable: true})) + // wisp -> durable edge: wisp_dependencies(issue_id -> wisps, target -> issues). + s.Require().NoError(deps.Insert(s.Ctx(), newDep(wisp, ids[0], types.DepBlocks), "tester", domain.DepInsertOpts{UseWispsTable: true})) + rec.reset() + + counts, err := comments.CountsByIssueIDs(s.Ctx(), ids, domain.CommentOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp CountsByIssueIDs", 1) + s.Equal(map[string]int{wisp: 2}, counts) + + list, err := comments.ListByIssueIDs(s.Ctx(), ids, domain.CommentOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp ListByIssueIDs", 1) + s.Require().Len(list, 1) + s.Require().Len(list[wisp], 2) + s.Equal("w0", list[wisp][0].Text) + + wl, err := labels.ListByIssueIDs(s.Ctx(), ids, domain.LabelOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp Labels.ListByIssueIDs", 1) + s.Equal(map[string][]string{wisp: {"wl"}}, wl) + + dl, err := deps.ListByIssueIDs(s.Ctx(), ids, domain.DepListOpts{Direction: domain.DepDirectionBoth, UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp Deps.ListByIssueIDs", 2) + s.Require().Len(dl.Outgoing[wisp], 1) + s.Equal(ids[0], dl.Outgoing[wisp][0].DependsOnID) + s.Require().Len(dl.Incoming[ids[0]], 1, "the durable target's inbound wisp edge lives in wisp_dependencies") + + dc, err := deps.CountsByIssueIDs(s.Ctx(), ids, domain.DepCountsOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp Deps.CountsByIssueIDs", 2) + s.Equal(&types.DependencyCounts{DependencyCount: 1}, dc[wisp]) + s.Equal(&types.DependencyCounts{DependentCount: 1}, dc[ids[0]]) + + issues := NewIssueSQLRepository(rec) + rec.reset() + got, err := issues.GetByIDs(s.Ctx(), ids, domain.IssueTableOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp GetByIDs", 1) + s.Require().Len(got, 1) + s.Equal(wisp, got[0].ID) + }) + + s.Run("GetByIDs", func() { + issues := NewIssueSQLRepository(rec) + rec.reset() + got, err := issues.GetByIDs(s.Ctx(), ids, domain.IssueTableOpts{}) + s.Require().NoError(err) + assertBatched("GetByIDs", 1) + s.Require().Len(got, len(seeded), "one row per seeded durable id, across all three batches") + gotIDs := make(map[string]bool, len(got)) + for _, iss := range got { + gotIDs[iss.ID] = true + } + for _, i := range seeded { + s.True(gotIDs[ids[i]], "%s present", ids[i]) + } + }) + + s.Run("FetchIssuesByIDs", func() { + // The residual reader behind export's SearchIssues(Limit=0): one + // unbounded IN list over every hit id before wy-sm01o2. + repo := &issueSQLRepositoryImpl{runner: rec} + rec.reset() + byID, err := repo.fetchIssuesByIDs(s.Ctx(), ids, issuesFilterTables, types.IssueFilter{SkipLabels: true}) + s.Require().NoError(err) + assertBatched("fetchIssuesByIDs", 1) + s.Require().Len(byID, len(seeded), "one row per seeded durable id, merged across all three batches") + for _, i := range seeded { + s.Require().NotNil(byID[ids[i]], "%s present in the merged map", ids[i]) + } + + // Hydration runs ONCE over the MERGED rows. Rows were scanned in all + // three batches, so a hydration moved inside the batch loop would + // issue three label reads here instead of one. + rec.reset() + byID, err = repo.fetchIssuesByIDs(s.Ctx(), ids, issuesFilterTables, types.IssueFilter{}) + s.Require().NoError(err) + calls := rec.reset() + s.Require().Len(calls, wantBatches+1, "%d fetch statements plus exactly one label hydration", wantBatches) + assertUnderCap("fetchIssuesByIDs (hydrating)", calls) + s.Contains(calls[len(calls)-1].query, "FROM labels", "the trailing statement is the single hydration") + for k, i := range seeded { + s.Require().NotNil(byID[ids[i]]) + s.Equal([]string{fmt.Sprintf("l%d", k)}, byID[ids[i]].Labels, "labels hydrated for %s", ids[i]) + } + }) + + s.Run("GetBlockingInfoOutbound", func() { + rec.reset() + info, err := deps.GetBlockingInfo(s.Ctx(), ids, domain.DepListOpts{}) + s.Require().NoError(err) + calls := rec.reset() + assertUnderCap("GetBlockingInfo (outbound)", calls) + s.Equal(wantBatches, countMatching(calls, "WHERE issue_id IN ("), "outbound leg: one statement per batch") + // ids[400] (batch 3) depends on ids[0] (batch 1), and ids[199] + // (batch 1) on ids[200] (batch 2): the edge is keyed by the + // depender, so both must appear in the merged map. + s.Require().Len(info.BlockedBy, 2, "the 'related' edge is not blocking and stays out") + s.Equal([]string{ids[0]}, info.BlockedBy[ids[n-1]]) + s.Equal([]string{ids[queryBatchSize]}, info.BlockedBy[ids[queryBatchSize-1]]) + s.Empty(info.Parent) + }) + + s.Run("GetBlockingInfoInbound", func() { + rec.reset() + info, err := deps.GetBlockingInfo(s.Ctx(), ids, domain.DepListOpts{}) + s.Require().NoError(err) + calls := rec.reset() + assertUnderCap("GetBlockingInfo (inbound)", calls) + s.Equal(wantBatches, countMatching(calls, "WHERE "+depTargetExpr+" IN ("), "inbound leg: one statement per batch") + // The inbound leg keys by the TARGET, which is the id the IN list + // constrains: ids[0] in batch 1 is blocked-target for a row whose + // other end sits in batch 3. + s.Require().Len(info.Blocks, 2) + s.Equal([]string{ids[n-1]}, info.Blocks[ids[0]]) + s.Equal([]string{ids[queryBatchSize-1]}, info.Blocks[ids[queryBatchSize]]) + }) + + s.Run("LoadStatusByID", func() { + repo := &dependencySQLRepositoryImpl{runner: rec} + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + rec.reset() + statusByID, err := repo.loadStatusByID(s.Ctx(), idSet) + s.Require().NoError(err) + calls := rec.reset() + assertUnderCap("loadStatusByID", calls) + s.Require().Len(calls, 2*wantBatches, "both planes, one statement per batch each") + s.Equal(wantBatches, countMatching(calls, "FROM issues WHERE id IN ("), "issues: one statement per batch") + s.Equal(wantBatches, countMatching(calls, "FROM wisps WHERE id IN ("), "wisps: one statement per batch") + // Rows from every batch AND from both planes merge into one map. + for _, i := range seeded { + s.Equal(types.StatusOpen, statusByID[ids[i]], "durable status for %s", ids[i]) + } + wisp := ids[queryBatchSize+50] + s.Equal(types.StatusOpen, statusByID[wisp], "the batch-2 wisp is read from the wisps table") + s.Len(statusByID, len(seeded)+1) + }) + + s.Run("LoadStatusByIDDuplicateAcrossTables", func() { + // Batching must not lose the cross-plane conflict: the id is in + // exactly one batch per table, so it is still recorded under issues + // before the wisps pass reaches it, whichever batch that is (the id + // set is a map, so its batch assignment is not fixed). + repo := &dependencySQLRepositoryImpl{runner: rec} + const dup = "bd-batch-dup" + s.seedIssueRow(dup) + s.seedWispRow(dup) + idSet := map[string]struct{}{dup: {}} + for _, id := range ids { + idSet[id] = struct{}{} + } + rec.reset() + _, err := repo.loadStatusByID(s.Ctx(), idSet) + s.Require().Error(err, "an id in both planes is still a conflict after batching") + s.Contains(err.Error(), "exists in both issues and wisps") + // The conflict aborts mid-run, so consume what it did issue (and + // hold it to the cap) rather than leaving it for the next subtest. + assertUnderCap("loadStatusByID (duplicate)", rec.reset()) + }) + + s.Run("EmptyInputIssuesNoStatement", func() { + _, err := comments.CountsByIssueIDs(s.Ctx(), nil, domain.CommentOpts{}) + s.Require().NoError(err) + _, err = labels.ListByIssueIDs(s.Ctx(), nil, domain.LabelOpts{}) + s.Require().NoError(err) + _, err = deps.CountsByIssueIDs(s.Ctx(), nil, domain.DepCountsOpts{}) + s.Require().NoError(err) + _, err = NewIssueSQLRepository(rec).GetByIDs(s.Ctx(), nil, domain.IssueTableOpts{}) + s.Require().NoError(err) + _, err = (&issueSQLRepositoryImpl{runner: rec}).fetchIssuesByIDs(s.Ctx(), nil, issuesFilterTables, types.IssueFilter{}) + s.Require().NoError(err) + _, err = deps.GetBlockingInfo(s.Ctx(), nil, domain.DepListOpts{}) + s.Require().NoError(err) + _, err = (&dependencySQLRepositoryImpl{runner: rec}).loadStatusByID(s.Ctx(), map[string]struct{}{}) + s.Require().NoError(err) + s.Empty(rec.reset()) + }) +} diff --git a/internal/storage/domain/db/comment.go b/internal/storage/domain/db/comment.go index 00e04f09a..dc2366b70 100644 --- a/internal/storage/domain/db/comment.go +++ b/internal/storage/domain/db/comment.go @@ -3,7 +3,6 @@ package db import ( "context" "fmt" - "strings" "time" "github.com/steveyegge/beads/internal/storage" @@ -31,76 +30,72 @@ func pickCommentTable(useWisps bool) string { func (r *commentSQLRepositoryImpl) CountsByIssueIDs(ctx context.Context, issueIDs []string, opts domain.CommentOpts) (map[string]int, error) { result := make(map[string]int) - if len(issueIDs) == 0 { - return result, nil - } - placeholders := make([]string, len(issueIDs)) - args := make([]any, len(issueIDs)) - for i, id := range issueIDs { - placeholders[i] = "?" - args[i] = id - } table := pickCommentTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf( - "SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) GROUP BY issue_id", - table, strings.Join(placeholders, ","), - ) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: %w", err) - } - defer rows.Close() - - for rows.Next() { - var issueID string - var count int - if err := rows.Scan(&issueID, &count); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: scan: %w", err) + err := forEachIDBatch(issueIDs, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf( + "SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) GROUP BY issue_id", + table, placeholders, + ) + rows, err := r.runner.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: %w", err) } - result[issueID] = count - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + var issueID string + var count int + if err := rows.Scan(&issueID, &count); err != nil { + return fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: scan: %w", err) + } + result[issueID] = count + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return result, nil } func (r *commentSQLRepositoryImpl) ListByIssueIDs(ctx context.Context, issueIDs []string, opts domain.CommentOpts) (map[string][]*types.Comment, error) { result := make(map[string][]*types.Comment) - if len(issueIDs) == 0 { - return result, nil - } - placeholders := make([]string, len(issueIDs)) - args := make([]any, len(issueIDs)) - for i, id := range issueIDs { - placeholders[i] = "?" - args[i] = id - } table := pickCommentTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf(` - SELECT id, issue_id, author, text, created_at - FROM %s - WHERE issue_id IN (%s) - ORDER BY issue_id, created_at ASC, id ASC - `, table, strings.Join(placeholders, ",")) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: %w", err) - } - defer rows.Close() - - for rows.Next() { - var c types.Comment - if err := rows.Scan(&c.ID, &c.IssueID, &c.Author, &c.Text, &c.CreatedAt); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: scan: %w", err) + err := forEachIDBatch(issueIDs, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf(` + SELECT id, issue_id, author, text, created_at + FROM %s + WHERE issue_id IN (%s) + ORDER BY issue_id, created_at ASC, id ASC + `, table, placeholders) + rows, err := r.runner.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: %w", err) } - cc := c - result[c.IssueID] = append(result[c.IssueID], &cc) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + var c types.Comment + if err := rows.Scan(&c.ID, &c.IssueID, &c.Author, &c.Text, &c.CreatedAt); err != nil { + return fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: scan: %w", err) + } + cc := c + result[c.IssueID] = append(result[c.IssueID], &cc) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return result, nil } diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 128202eb7..53f09dffa 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -414,71 +414,75 @@ func (r *dependencySQLRepositoryImpl) ListByIssueIDs(ctx context.Context, issueI Outgoing: make(map[string][]*types.Dependency), Incoming: make(map[string][]*types.Dependency), } - if len(issueIDs) == 0 { - return result, nil - } - - idPlaceholders, idArgs := buildInPlaceholders(issueIDs) typeWhere, typeArgs := buildTypeFilter(opts.Types) table := pickDepTable(opts.UseWispsTable) - if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionOut { - //nolint:gosec // G201: table and depSelectColumns are hardcoded - q := fmt.Sprintf( - `SELECT %s FROM %s WHERE issue_id IN (%s)%s ORDER BY issue_id`, - depSelectColumns, table, idPlaceholders, typeWhere, - ) - args := combineArgs(idArgs, typeArgs) - if err := r.queryDeps(ctx, q, args, result.Outgoing, true); err != nil { - return domain.DepBulkResult{}, fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (out): %w", err) - } - } + err := forEachIDBatch(issueIDs, func(batch []string) error { + idPlaceholders, idArgs := buildInPlaceholders(batch) - if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionIn { - //nolint:gosec // G201: table, depSelectColumns, depTargetExpr are hardcoded - q := fmt.Sprintf( - `SELECT %s FROM %s WHERE %s IN (%s)%s ORDER BY issue_id`, - depSelectColumns, table, depTargetExpr, idPlaceholders, typeWhere, - ) - args := combineArgs(idArgs, typeArgs) - if err := r.queryDeps(ctx, q, args, result.Incoming, false); err != nil { - return domain.DepBulkResult{}, fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (in): %w", err) + if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionOut { + //nolint:gosec // G201: table and depSelectColumns are hardcoded + q := fmt.Sprintf( + `SELECT %s FROM %s WHERE issue_id IN (%s)%s ORDER BY issue_id`, + depSelectColumns, table, idPlaceholders, typeWhere, + ) + args := combineArgs(idArgs, typeArgs) + if err := r.queryDeps(ctx, q, args, result.Outgoing, true); err != nil { + return fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (out): %w", err) + } } - } + if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionIn { + //nolint:gosec // G201: table, depSelectColumns, depTargetExpr are hardcoded + q := fmt.Sprintf( + `SELECT %s FROM %s WHERE %s IN (%s)%s ORDER BY issue_id`, + depSelectColumns, table, depTargetExpr, idPlaceholders, typeWhere, + ) + args := combineArgs(idArgs, typeArgs) + if err := r.queryDeps(ctx, q, args, result.Incoming, false); err != nil { + return fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (in): %w", err) + } + } + return nil + }) + if err != nil { + return domain.DepBulkResult{}, err + } return result, nil } func (r *dependencySQLRepositoryImpl) CountsByIssueIDs(ctx context.Context, issueIDs []string, opts domain.DepCountsOpts) (map[string]*types.DependencyCounts, error) { result := make(map[string]*types.DependencyCounts) - if len(issueIDs) == 0 { - return result, nil - } for _, id := range issueIDs { result[id] = &types.DependencyCounts{} } - - idPlaceholders, idArgs := buildInPlaceholders(issueIDs) table := pickDepTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - outQ := fmt.Sprintf( - `SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) AND type = 'blocks' GROUP BY issue_id`, - table, idPlaceholders, - ) - if err := scanCounts(ctx, r.runner, outQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependencyCount = n }); err != nil { - return nil, fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (out): %w", err) - } + err := forEachIDBatch(issueIDs, func(batch []string) error { + idPlaceholders, idArgs := buildInPlaceholders(batch) - //nolint:gosec // G201: table and depTargetExpr are hardcoded - inQ := fmt.Sprintf( - `SELECT %s AS depends_on_id, COUNT(*) FROM %s WHERE %s IN (%s) AND type = 'blocks' GROUP BY %s`, - depTargetExpr, table, depTargetExpr, idPlaceholders, depTargetExpr, - ) - if err := scanCounts(ctx, r.runner, inQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependentCount = n }); err != nil { - return nil, fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (in): %w", err) - } + //nolint:gosec // G201: table is one of two hardcoded constants + outQ := fmt.Sprintf( + `SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) AND type = 'blocks' GROUP BY issue_id`, + table, idPlaceholders, + ) + if err := scanCounts(ctx, r.runner, outQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependencyCount = n }); err != nil { + return fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (out): %w", err) + } + //nolint:gosec // G201: table and depTargetExpr are hardcoded + inQ := fmt.Sprintf( + `SELECT %s AS depends_on_id, COUNT(*) FROM %s WHERE %s IN (%s) AND type = 'blocks' GROUP BY %s`, + depTargetExpr, table, depTargetExpr, idPlaceholders, depTargetExpr, + ) + if err := scanCounts(ctx, r.runner, inQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependentCount = n }); err != nil { + return fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (in): %w", err) + } + return nil + }) + if err != nil { + return nil, err + } return result, nil } @@ -493,26 +497,41 @@ func (r *dependencySQLRepositoryImpl) GetBlockingInfo(ctx context.Context, issue } table := pickDepTable(opts.UseWispsTable) - idPlaceholders, idArgs := buildInPlaceholders(issueIDs) - //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - outQ := fmt.Sprintf( - "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE issue_id IN (%s) AND type IN ('blocks', 'parent-child')", - depTargetExpr, table, idPlaceholders, - ) - outRows, err := r.scanBlockingRows(ctx, outQ, idArgs) - if err != nil { - return domain.BlockingInfo{}, fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: outbound: %w", err) - } + // Both legs are batched at queryBatchSize and their rows concatenated. + // Each leg keys its result by a column the IN list constrains — outbound + // by issue_id, inbound by the dependency target — and an id lands in + // exactly one batch, so every row that shares a key comes from the same + // batch and the merge below is unchanged by the split. + var outRows, inRows []blockingRow + err := forEachIDBatch(issueIDs, func(batch []string) error { + idPlaceholders, idArgs := buildInPlaceholders(batch) - //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - inQ := fmt.Sprintf( - "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE %s IN (%s) AND type = 'blocks'", - depTargetExpr, table, depTargetExpr, idPlaceholders, - ) - inRows, err := r.scanBlockingRows(ctx, inQ, idArgs) + //nolint:gosec // G201: table and depTargetExpr are hardcoded constants + outQ := fmt.Sprintf( + "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE issue_id IN (%s) AND type IN ('blocks', 'parent-child')", + depTargetExpr, table, idPlaceholders, + ) + batchOut, err := r.scanBlockingRows(ctx, outQ, idArgs) + if err != nil { + return fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: outbound: %w", err) + } + outRows = append(outRows, batchOut...) + + //nolint:gosec // G201: table and depTargetExpr are hardcoded constants + inQ := fmt.Sprintf( + "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE %s IN (%s) AND type = 'blocks'", + depTargetExpr, table, depTargetExpr, idPlaceholders, + ) + batchIn, err := r.scanBlockingRows(ctx, inQ, idArgs) + if err != nil { + return fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: inbound: %w", err) + } + inRows = append(inRows, batchIn...) + return nil + }) if err != nil { - return domain.BlockingInfo{}, fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: inbound: %w", err) + return domain.BlockingInfo{}, err } statusIDs := make(map[string]struct{}) @@ -607,12 +626,19 @@ func (r *dependencySQLRepositoryImpl) loadStatusByID(ctx context.Context, idSet for id := range idSet { ids = append(ids, id) } - placeholders, args := buildInPlaceholders(ids) sourceByID := make(map[string]string, len(idSet)) + // Each table is read in ceil(len(ids)/queryBatchSize) statements. The + // cross-table duplicate check is unaffected: an id lands in exactly one + // batch per table, so it is still recorded under `issues` before the + // `wisps` pass reaches it. for _, table := range []string{"issues", "wisps"} { - //nolint:gosec // G201: table is a hardcoded constant - q := fmt.Sprintf("SELECT id, status FROM %s WHERE id IN (%s)", table, placeholders) - if err := r.scanStatusRows(ctx, q, args, table, statusByID, sourceByID); err != nil { + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is a hardcoded constant + q := fmt.Sprintf("SELECT id, status FROM %s WHERE id IN (%s)", table, placeholders) + return r.scanStatusRows(ctx, q, args, table, statusByID, sourceByID) + }) + if err != nil { return nil, err } } diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 41962ef48..1f08abed5 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -567,35 +567,33 @@ func (r *issueSQLRepositoryImpl) Get(ctx context.Context, id string, opts domain } func (r *issueSQLRepositoryImpl) GetByIDs(ctx context.Context, ids []string, opts domain.IssueTableOpts) ([]*types.Issue, error) { - if len(ids) == 0 { - return nil, nil - } - placeholders := make([]string, len(ids)) - args := make([]any, len(ids)) - for i, id := range ids { - placeholders[i] = "?" - args[i] = id - } table := pickIssueTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf("SELECT %s FROM %s %s WHERE id IN (%s)", - issueSelectColumns, table, sqlbuild.LeaseJoin(table), strings.Join(placeholders, ",")) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: GetByIDs: %w", err) - } - defer rows.Close() - var out []*types.Issue - for rows.Next() { - issue, err := scanIssue(rows) + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf("SELECT %s FROM %s %s WHERE id IN (%s)", + issueSelectColumns, table, sqlbuild.LeaseJoin(table), placeholders) + rows, err := r.runner.QueryContext(ctx, q, args...) if err != nil { - return nil, fmt.Errorf("db: GetByIDs: scan: %w", err) + return fmt.Errorf("db: GetByIDs: %w", err) } - out = append(out, issue) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: GetByIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + issue, err := scanIssue(rows) + if err != nil { + return fmt.Errorf("db: GetByIDs: scan: %w", err) + } + out = append(out, issue) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: GetByIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return out, nil } diff --git a/internal/storage/domain/db/issue_search.go b/internal/storage/domain/db/issue_search.go index f9122695c..c849668f7 100644 --- a/internal/storage/domain/db/issue_search.go +++ b/internal/storage/domain/db/issue_search.go @@ -182,33 +182,45 @@ func (r *issueSQLRepositoryImpl) fetchIssuesByIDs(ctx context.Context, ids []str return map[string]*types.Issue{}, nil } - placeholders, args := buildInPlaceholders(ids) + out := make(map[string]*types.Issue, len(ids)) + ordered := make([]*types.Issue, 0, len(ids)) + // One statement per batch, scan order concatenated across batches: an id + // lands in exactly one batch, so the merged map is what a single + // statement over all ids would have produced. The fetch itself carries no + // ORDER BY (callers order by their own id list, or by src), so the + // concatenation is as ordered as the single statement was. + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) - //nolint:gosec // G201: tables.Main is "issues" or "wisps"; placeholders are ?. - fetchSQL := fmt.Sprintf(`SELECT %s FROM %s %s WHERE id IN (%s)`, - issueSelectColumns, tables.Main, sqlbuild.LeaseJoin(tables.Main), placeholders) - rows, err := r.runner.QueryContext(ctx, fetchSQL, args...) + //nolint:gosec // G201: tables.Main is "issues" or "wisps"; placeholders are ?. + fetchSQL := fmt.Sprintf(`SELECT %s FROM %s %s WHERE id IN (%s)`, + issueSelectColumns, tables.Main, sqlbuild.LeaseJoin(tables.Main), placeholders) + rows, err := r.runner.QueryContext(ctx, fetchSQL, args...) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + issue, scanErr := scanIssue(rows) + if scanErr != nil { + return fmt.Errorf("scan: %w", scanErr) + } + out[issue.ID] = issue + ordered = append(ordered, issue) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("rows: %w", err) + } + return nil + }) if err != nil { return nil, err } - out := make(map[string]*types.Issue, len(ids)) - ordered := make([]*types.Issue, 0, len(ids)) - for rows.Next() { - issue, scanErr := scanIssue(rows) - if scanErr != nil { - _ = rows.Close() - return nil, fmt.Errorf("scan: %w", scanErr) - } - out[issue.ID] = issue - ordered = append(ordered, issue) - } - - _ = rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("rows: %w", err) - } - + // Hydration runs ONCE over every scanned row: hydrateIssues batches its + // own relation reads (wy-237yfi), so per-batch hydration would only add + // round trips. if err := r.hydrateIssues(ctx, ordered, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil { return nil, fmt.Errorf("hydrate: %w", err) } @@ -361,30 +373,29 @@ func (r *issueSQLRepositoryImpl) hydrateIssues(ctx context.Context, issues []*ty //nolint:gosec // G201: labelTable is "labels" or "wisp_labels" (hardcoded by callers). func (r *issueSQLRepositoryImpl) getLabelsFromTable(ctx context.Context, labelTable string, ids []string) (map[string][]string, error) { result := make(map[string][]string) - for start := 0; start < len(ids); start += queryBatchSize { - end := start + queryBatchSize - if end > len(ids) { - end = len(ids) - } - placeholders, args := buildInPlaceholders(ids[start:end]) + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) rows, err := r.runner.QueryContext(ctx, fmt.Sprintf( `SELECT issue_id, label FROM %s WHERE issue_id IN (%s) ORDER BY issue_id, label`, labelTable, placeholders), args...) if err != nil { - return nil, fmt.Errorf("get labels from %s: %w", labelTable, err) + return fmt.Errorf("get labels from %s: %w", labelTable, err) } + defer rows.Close() for rows.Next() { var issueID, label string if err := rows.Scan(&issueID, &label); err != nil { - _ = rows.Close() - return nil, fmt.Errorf("get labels: scan: %w", err) + return fmt.Errorf("get labels: scan: %w", err) } result[issueID] = append(result[issueID], label) } - _ = rows.Close() if err := rows.Err(); err != nil { - return nil, fmt.Errorf("get labels: rows: %w", err) + return fmt.Errorf("get labels: rows: %w", err) } + return nil + }) + if err != nil { + return nil, err } return result, nil } @@ -392,31 +403,30 @@ func (r *issueSQLRepositoryImpl) getLabelsFromTable(ctx context.Context, labelTa //nolint:gosec // G201: depTable is "dependencies" or "wisp_dependencies" (hardcoded by callers). func (r *issueSQLRepositoryImpl) getDependencyRecordsFromTable(ctx context.Context, depTable string, ids []string) (map[string][]*types.Dependency, error) { result := make(map[string][]*types.Dependency) - for start := 0; start < len(ids); start += queryBatchSize { - end := start + queryBatchSize - if end > len(ids) { - end = len(ids) - } - placeholders, args := buildInPlaceholders(ids[start:end]) + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) rows, err := r.runner.QueryContext(ctx, fmt.Sprintf( `SELECT issue_id, %s AS depends_on_id, type, created_at, created_by, metadata, thread_id FROM %s WHERE issue_id IN (%s) ORDER BY issue_id`, depTargetExpr, depTable, placeholders), args...) if err != nil { - return nil, fmt.Errorf("get dep records from %s: %w", depTable, err) + return fmt.Errorf("get dep records from %s: %w", depTable, err) } + defer rows.Close() for rows.Next() { dep, scanErr := scanDepRow(rows) if scanErr != nil { - _ = rows.Close() - return nil, fmt.Errorf("get dep records: scan: %w", scanErr) + return fmt.Errorf("get dep records: scan: %w", scanErr) } result[dep.IssueID] = append(result[dep.IssueID], dep) } - _ = rows.Close() if err := rows.Err(); err != nil { - return nil, fmt.Errorf("get dep records: rows: %w", err) + return fmt.Errorf("get dep records: rows: %w", err) } + return nil + }) + if err != nil { + return nil, err } return result, nil } diff --git a/internal/storage/domain/db/issue_search_counts.go b/internal/storage/domain/db/issue_search_counts.go index 5a617cfc1..f04f63b0e 100644 --- a/internal/storage/domain/db/issue_search_counts.go +++ b/internal/storage/domain/db/issue_search_counts.go @@ -136,15 +136,11 @@ func readyHydrationFor(filter types.WorkFilter) sqlbuild.CountsHydration { // within per-statement limits (mirrors issueops.runReadyCountsInTx). func (r *issueSQLRepositoryImpl) fetchCountsByIDs(ctx context.Context, ids []string, tables filterTables, includeWispReverseDeps bool, hyd sqlbuild.CountsHydration) (map[string]*types.IssueWithCounts, error) { out := make(map[string]*types.IssueWithCounts, len(ids)) - for start := 0; start < len(ids); start += queryBatchSize { - end := start + queryBatchSize - if end > len(ids) { - end = len(ids) - } - countsSQL, args := sqlbuild.SearchCountsSQL(tables, ids[start:end], "", "", "", includeWispReverseDeps, hyd) + err := forEachIDBatch(ids, func(batch []string) error { + countsSQL, args := sqlbuild.SearchCountsSQL(tables, batch, "", "", "", includeWispReverseDeps, hyd) items, err := r.scanCountsQuery(ctx, tables, countsSQL, args, hyd) if err != nil { - return nil, err + return err } for _, iwc := range items { if iwc == nil || iwc.Issue == nil { @@ -152,6 +148,10 @@ func (r *issueSQLRepositoryImpl) fetchCountsByIDs(ctx context.Context, ids []str } out[iwc.Issue.ID] = iwc } + return nil + }) + if err != nil { + return nil, err } return out, nil } diff --git a/internal/storage/domain/db/label.go b/internal/storage/domain/db/label.go index 14e73a80c..8f196b873 100644 --- a/internal/storage/domain/db/label.go +++ b/internal/storage/domain/db/label.go @@ -152,36 +152,34 @@ func (r *labelSQLRepositoryImpl) List(ctx context.Context, issueID string, opts func (r *labelSQLRepositoryImpl) ListByIssueIDs(ctx context.Context, issueIDs []string, opts domain.LabelOpts) (map[string][]string, error) { result := make(map[string][]string) - if len(issueIDs) == 0 { - return result, nil - } - placeholders := make([]string, len(issueIDs)) - args := make([]any, len(issueIDs)) - for i, id := range issueIDs { - placeholders[i] = "?" - args[i] = id - } table := pickLabelTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf( - "SELECT issue_id, label FROM %s WHERE issue_id IN (%s) ORDER BY issue_id, label", - table, strings.Join(placeholders, ","), - ) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: %w", err) - } - defer rows.Close() - - for rows.Next() { - var issueID, label string - if err := rows.Scan(&issueID, &label); err != nil { - return nil, fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: scan: %w", err) + err := forEachIDBatch(issueIDs, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf( + "SELECT issue_id, label FROM %s WHERE issue_id IN (%s) ORDER BY issue_id, label", + table, placeholders, + ) + rows, err := r.runner.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: %w", err) } - result[issueID] = append(result[issueID], label) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + var issueID, label string + if err := rows.Scan(&issueID, &label); err != nil { + return fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: scan: %w", err) + } + result[issueID] = append(result[issueID], label) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return result, nil } diff --git a/internal/storage/domain/issue_delete.go b/internal/storage/domain/issue_delete.go index 32c77148f..832cfa9f6 100644 --- a/internal/storage/domain/issue_delete.go +++ b/internal/storage/domain/issue_delete.go @@ -287,8 +287,12 @@ func (u *issueUseCaseImpl) previewDelete(ctx context.Context, ids []string) (Del for _, iss := range fromIssues { preview.Issues[iss.ID] = iss } + // Only the WISPS table is optional here: this read is `FROM wisps LEFT + // JOIN leases`, so a blanket table-not-exist check reported a rig missing + // `leases` as a rig with no wisp plane and listed live wisps under + // NotFound (the WispPlaneIDs lesson, wy-237yfi). fromWisps, err := u.issueRepo.GetByIDs(ctx, ids, IssueTableOpts{UseWispsTable: true}) - if err != nil && !dberrors.IsTableNotExist(err) { + if err != nil && !dberrors.IsMissingTable(err, "wisps") { return preview, fmt.Errorf("previewDelete: load wisps: %w", err) } for _, iss := range fromWisps { @@ -383,8 +387,11 @@ func (u *issueUseCaseImpl) collectConnectedIssues( for _, iss := range fromIssues { out[iss.ID] = iss } + // As in previewDelete: the wisp read joins `leases`, and only a missing + // `wisps` table means "this rig has no wisp plane". Anything else left + // the neighbors unhydrated behind a nil error. fromWisps, err := u.issueRepo.GetByIDs(ctx, ids, IssueTableOpts{UseWispsTable: true}) - if err != nil && !dberrors.IsTableNotExist(err) { + if err != nil && !dberrors.IsMissingTable(err, "wisps") { return nil, nil, fmt.Errorf("hydrate neighbors (wisps): %w", err) } for _, iss := range fromWisps { diff --git a/internal/storage/domain/issue_delete_missing_table_test.go b/internal/storage/domain/issue_delete_missing_table_test.go new file mode 100644 index 000000000..d068aa368 --- /dev/null +++ b/internal/storage/domain/issue_delete_missing_table_test.go @@ -0,0 +1,153 @@ +package domain + +import ( + "context" + "errors" + "fmt" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/types" +) + +// deleteMissingTable builds the error a Dolt read raises when `table` is +// absent, in the go-mysql-server wording dberrors.MissingTableName parses. +func deleteMissingTable(table string) error { + return &mysql.MySQLError{Number: 1146, Message: "table not found: " + table} +} + +// stubDeleteIssueRepo answers only the reads the delete-preview path makes. +// The embedded interface is nil on purpose: a call this test did not intend +// panics rather than passing quietly. +type stubDeleteIssueRepo struct { + IssueSQLRepository + durable []*types.Issue + wisps []*types.Issue + wispErr error + dependents []string +} + +func (r *stubDeleteIssueRepo) GetByIDs(_ context.Context, _ []string, opts IssueTableOpts) ([]*types.Issue, error) { + if opts.UseWispsTable { + if r.wispErr != nil { + // The real reader wraps, and the callers classify through the wrap. + return nil, fmt.Errorf("db: GetByIDs: %w", r.wispErr) + } + return r.wisps, nil + } + return r.durable, nil +} + +func (r *stubDeleteIssueRepo) FindAllDependents(_ context.Context, _ []string) ([]string, error) { + return r.dependents, nil +} + +// stubDeleteDepRepo returns the same edge set for the durable plane and +// nothing for the wisp plane. +type stubDeleteDepRepo struct { + DependencySQLRepository + durable DepBulkResult +} + +func (r *stubDeleteDepRepo) ListByIssueIDs(_ context.Context, _ []string, opts DepListOpts) (DepBulkResult, error) { + if opts.UseWispsTable { + return DepBulkResult{}, nil + } + return r.durable, nil +} + +func deleteUseCase(issueRepo IssueSQLRepository, depRepo DependencySQLRepository) *issueUseCaseImpl { + return &issueUseCaseImpl{issueRepo: issueRepo, depRepo: depRepo} +} + +// TestPreviewDeleteBrokenWispPlaneIsAnError pins wy-sm01o2's half of the +// wy-237yfi lesson on the delete path: previewDelete's wisp read is +// `FROM wisps LEFT JOIN leases`, so a blanket table-not-exist tolerance +// reported a rig missing `leases` as a rig with no wisp plane and listed live +// wisps under NotFound. +func TestPreviewDeleteBrokenWispPlaneIsAnError(t *testing.T) { + for _, missing := range []string{"leases", "wisp_labels"} { + t.Run(missing, func(t *testing.T) { + gone := deleteMissingTable(missing) + u := deleteUseCase(&stubDeleteIssueRepo{wispErr: gone}, &stubDeleteDepRepo{}) + + _, err := u.previewDelete(t.Context(), []string{"bd-1"}) + if !errors.Is(err, gone) { + t.Fatalf("previewDelete hid a broken wisp plane: %v", err) + } + }) + } +} + +// TestPreviewDeleteMissingWispsTableIsTolerated is the control: a +// pre-migration rig really has no wisps table, and delete still previews. +func TestPreviewDeleteMissingWispsTableIsTolerated(t *testing.T) { + u := deleteUseCase( + &stubDeleteIssueRepo{wispErr: deleteMissingTable("wisps")}, + &stubDeleteDepRepo{}, + ) + + preview, err := u.previewDelete(t.Context(), []string{"bd-1"}) + if err != nil { + t.Fatalf("previewDelete errored on a rig with no wisps table: %v", err) + } + if len(preview.NotFound) != 1 || preview.NotFound[0] != "bd-1" { + t.Fatalf("NotFound = %v, want [bd-1]", preview.NotFound) + } +} + +// connectedOverBrokenWispPlane runs collectConnectedIssues with one durable +// edge, so the neighbour hydration below actually runs, and fails its wisp +// leg with wispErr. +func connectedOverBrokenWispPlane(t *testing.T, wispErr error) (map[string]*types.Issue, error) { + t.Helper() + deps := &stubDeleteDepRepo{durable: DepBulkResult{ + Outgoing: map[string][]*types.Dependency{ + "bd-1": {{IssueID: "bd-1", DependsOnID: "bd-2", Type: types.DepBlocks}}, + }, + }} + u := deleteUseCase(&stubDeleteIssueRepo{wispErr: wispErr}, deps) + out, _, err := u.collectConnectedIssues(t.Context(), []string{"bd-1"}, map[string]bool{"bd-1": true}) + return out, err +} + +// TestCollectConnectedBrokenWispPlaneIsAnError is the second GetByIDs(wisps) +// site: swallowing anything but a missing `wisps` table left the neighbours +// unhydrated behind a nil error. +func TestCollectConnectedBrokenWispPlaneIsAnError(t *testing.T) { + for _, missing := range []string{"leases", "wisp_labels"} { + t.Run(missing, func(t *testing.T) { + gone := deleteMissingTable(missing) + if _, err := connectedOverBrokenWispPlane(t, gone); !errors.Is(err, gone) { + t.Fatalf("hydrate neighbors hid a broken wisp plane: %v", err) + } + }) + } +} + +// TestCollectConnectedMissingWispsTableIsTolerated is that site's control. +func TestCollectConnectedMissingWispsTableIsTolerated(t *testing.T) { + out, err := connectedOverBrokenWispPlane(t, deleteMissingTable("wisps")) + if err != nil { + t.Fatalf("hydrate neighbors errored on a rig with no wisps table: %v", err) + } + if len(out) != 0 { + t.Fatalf("got %d hydrated neighbours, want 0", len(out)) + } +} + +// TestDeleteWispReadUnrelatedErrorPropagates is the fail-open control both +// sites share: a tolerance that cannot parse a table name out of an error +// must not treat it as an absent wisp plane. +func TestDeleteWispReadUnrelatedErrorPropagates(t *testing.T) { + boom := errors.New("connection refused") + + u := deleteUseCase(&stubDeleteIssueRepo{wispErr: boom}, &stubDeleteDepRepo{}) + if _, err := u.previewDelete(t.Context(), []string{"bd-1"}); !errors.Is(err, boom) { + t.Fatalf("previewDelete did not propagate a failed wisp read: %v", err) + } + if _, err := connectedOverBrokenWispPlane(t, boom); !errors.Is(err, boom) { + t.Fatalf("hydrate neighbors did not propagate a failed wisp read: %v", err) + } +}