beads/internal/storage/issueops/bulk_ops.go
Harry be65f59cd0 fix(dolt): batch wisp-ID partition in bulk hydrators (GH#3414)
Every mutating bd command against a remote Dolt backend hung ~10–15s
with `auto-export failed: ... hydrate labels: ... context canceled`.

Root cause: five bulk fetchers in internal/storage/issueops partitioned
IDs into wisp vs permanent buckets by calling IsActiveWispInTx once per
ID, and each call is a round-trip to the SQL server. With ~600 issues
over WAN latency that's ~12s of sequential SELECTs *before* the real
label query runs — blowing past the context deadline.

Fix:
- Add PartitionWispIDsInTx that batches the check into one
  `SELECT id FROM wisps WHERE id IN (...)` per queryBatchSize chunk,
  reducing cost from O(N) to O(1 + N/200) round-trips. Falls back to
  "no wisps" when the wisps table is absent (pre-migration dbs).
- Rewrite GetLabelsForIssuesInTx, GetCommentsForIssuesInTx,
  GetDependencyRecordsForIssuesInTx, GetBlockingInfoForIssuesInTx, and
  DeleteIssuesInTx to use it.
- Add fast-path helpers GetLabelsForIssuesFromTableInTx and
  GetDependencyRecordsForIssuesFromTableInTx for callers (searchTableInTx)
  that already know every ID is in a single table, skipping the partition
  round-trip entirely on the hot auto-export path.

Tests:
- New wisp_partition_test.go covering the partitioner directly: empty,
  all-perm, all-wisp, mixed-with-unknown, input-order preservation, and
  a 450-ID large-batch case crossing the queryBatchSize boundary.
- End-to-end mixed-routing regression tests for GetCommentsForIssues,
  GetDependencyRecordsForIssues, and a 250-ID GetLabelsForIssues case
  that crosses the chunk boundary.

Made-with: Cursor
2026-05-02 13:27:24 -07:00

450 lines
13 KiB
Go

package issueops
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
)
// GetIssueByExternalRefInTx looks up an issue by its external_ref field.
// Checks both issues and wisps tables so that pushed ephemeral beads are
// found during pull dedup.
// Returns the issue ID if found. Returns storage.ErrNotFound (wrapped) if not found.
func GetIssueByExternalRefInTx(ctx context.Context, tx *sql.Tx, externalRef string) (string, error) {
var id string
err := tx.QueryRowContext(ctx, "SELECT id FROM issues WHERE external_ref = ?", externalRef).Scan(&id)
if err == nil {
return id, nil
}
if err != sql.ErrNoRows {
return "", fmt.Errorf("get issue by external_ref: %w", err)
}
// Fall through to wisps table — pushed wisps have external_ref set there.
err = tx.QueryRowContext(ctx, "SELECT id FROM wisps WHERE external_ref = ?", externalRef).Scan(&id)
if err == sql.ErrNoRows {
return "", fmt.Errorf("%w: external_ref %s", storage.ErrNotFound, externalRef)
}
if err != nil {
if isTableNotExistError(err) {
return "", fmt.Errorf("%w: external_ref %s", storage.ErrNotFound, externalRef)
}
return "", fmt.Errorf("get wisp by external_ref: %w", err)
}
return id, nil
}
// GetIssuesByLabelInTx returns issue IDs matching a label from both issues and wisps tables.
//
//nolint:gosec // G201: tables are hardcoded
func GetIssuesByLabelInTx(ctx context.Context, tx *sql.Tx, label string) ([]string, error) {
var ids []string
rows, err := tx.QueryContext(ctx, `
SELECT i.id FROM issues i
JOIN labels l ON i.id = l.issue_id
WHERE l.label = ?
ORDER BY i.priority ASC, i.created_at DESC
`, label)
if err != nil {
return nil, fmt.Errorf("get issues by label: %w", err)
}
defer rows.Close()
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan issue id: %w", err)
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate issues by label: %w", err)
}
wispRows, err := tx.QueryContext(ctx, `SELECT issue_id FROM wisp_labels WHERE label = ?`, label)
if err != nil {
return nil, fmt.Errorf("get wisp issues by label: %w", err)
}
defer wispRows.Close()
for wispRows.Next() {
var id string
if err := wispRows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan wisp issue id: %w", err)
}
ids = append(ids, id)
}
if err := wispRows.Err(); err != nil {
return nil, fmt.Errorf("iterate wisp issues by label: %w", err)
}
return ids, nil
}
// DeleteConfigInTx removes a configuration value.
func DeleteConfigInTx(ctx context.Context, tx *sql.Tx, key string) error {
_, err := tx.ExecContext(ctx, "DELETE FROM config WHERE `key` = ?", key)
if err != nil {
return fmt.Errorf("delete config %s: %w", key, err)
}
return nil
}
// GetCommentsForIssuesInTx retrieves comments for multiple issues, partitioning
// between comments and wisp_comments tables.
//
//nolint:gosec // G201: table is hardcoded
func GetCommentsForIssuesInTx(ctx context.Context, tx *sql.Tx, issueIDs []string) (map[string][]*types.Comment, error) {
if len(issueIDs) == 0 {
return make(map[string][]*types.Comment), nil
}
result := make(map[string][]*types.Comment)
// Partition IDs by wisp status in a single batched query, to avoid N
// round-trips on remote backends (GH#3414).
wispIDs, permIDs, err := PartitionWispIDsInTx(ctx, tx, issueIDs)
if err != nil {
return nil, err
}
if len(permIDs) > 0 {
if err := getCommentsForIDsInto(ctx, tx, "comments", permIDs, result); err != nil {
return nil, err
}
}
if len(wispIDs) > 0 {
if err := getCommentsForIDsInto(ctx, tx, "wisp_comments", wispIDs, result); err != nil {
return nil, err
}
}
return result, nil
}
//nolint:gosec // G201: table is hardcoded
func getCommentsForIDsInto(ctx context.Context, tx *sql.Tx, table string, ids []string, result map[string][]*types.Comment) error {
for start := 0; start < len(ids); start += queryBatchSize {
end := start + queryBatchSize
if end > len(ids) {
end = len(ids)
}
batch := ids[start:end]
placeholders, args := buildSQLInClause(batch)
query := fmt.Sprintf(`
SELECT id, issue_id, author, text, created_at
FROM %s
WHERE issue_id IN (%s)
ORDER BY issue_id, created_at ASC, id ASC
`, table, placeholders)
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return fmt.Errorf("get comments from %s: %w", table, err)
}
for rows.Next() {
var c types.Comment
if err := rows.Scan(&c.ID, &c.IssueID, &c.Author, &c.Text, &c.CreatedAt); err != nil {
_ = rows.Close()
return fmt.Errorf("scan comment: %w", err)
}
result[c.IssueID] = append(result[c.IssueID], &c)
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
}
return nil
}
// DeleteIssuesBySourceRepoInTx removes all issues from a source repo and their related data.
//
//nolint:gosec // G201: table is validated by hardcoded list
func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo string) (int, error) {
rows, err := tx.QueryContext(ctx, `SELECT id FROM issues WHERE source_repo = ?`, sourceRepo)
if err != nil {
return 0, fmt.Errorf("query issues: %w", err)
}
var issueIDs []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
_ = rows.Close()
return 0, fmt.Errorf("scan issue ID: %w", err)
}
issueIDs = append(issueIDs, id)
}
_ = rows.Close()
if len(issueIDs) == 0 {
return 0, nil
}
for _, table := range []string{"dependencies", "events", "comments", "labels"} {
for _, id := range issueIDs {
if table == "dependencies" {
_, err = tx.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE issue_id = ? OR depends_on_id = ?", table), id, id)
} else {
_, err = tx.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE issue_id = ?", table), id)
}
if err != nil {
return 0, fmt.Errorf("delete from %s for %s: %w", table, id, err)
}
}
}
result, err := tx.ExecContext(ctx, `DELETE FROM issues WHERE source_repo = ?`, sourceRepo)
if err != nil {
return 0, fmt.Errorf("delete issues: %w", err)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return 0, fmt.Errorf("rows affected: %w", err)
}
return int(rowsAffected), nil
}
// UpdateIssueIDInTx renames an issue and updates all references across tables.
//
//nolint:gosec // G201: table names are hardcoded
func UpdateIssueIDInTx(ctx context.Context, tx *sql.Tx, oldID, newID string, issue *types.Issue, actor string) error {
isWisp := IsActiveWispInTx(ctx, tx, oldID)
if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS = 0`); err != nil {
return fmt.Errorf("disable FK checks: %w", err)
}
defer func() { _, _ = tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS = 1`) }()
if isWisp {
return updateWispIDInTx(ctx, tx, oldID, newID, issue, actor)
}
return updateIssueIDInTx(ctx, tx, oldID, newID, issue, actor)
}
func updateIssueIDInTx(ctx context.Context, tx *sql.Tx, oldID, newID string, issue *types.Issue, actor string) error {
now := time.Now().UTC()
result, err := tx.ExecContext(ctx, `
UPDATE issues
SET id = ?, title = ?, description = ?, design = ?, acceptance_criteria = ?, notes = ?, updated_at = ?
WHERE id = ?
`, newID, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes, now, oldID)
if err != nil {
return fmt.Errorf("update issue ID: %w", err)
}
if rows, _ := result.RowsAffected(); rows == 0 {
return fmt.Errorf("issue not found: %s", oldID)
}
refs := []struct{ table, col string }{
{"dependencies", "issue_id"},
{"dependencies", "depends_on_id"},
{"events", "issue_id"},
{"labels", "issue_id"},
{"comments", "issue_id"},
{"issue_snapshots", "issue_id"},
{"compaction_snapshots", "issue_id"},
{"child_counters", "parent_id"},
{"wisp_dependencies", "issue_id"},
{"wisp_dependencies", "depends_on_id"},
{"wisp_events", "issue_id"},
{"wisp_labels", "issue_id"},
{"wisp_comments", "issue_id"},
}
for _, r := range refs {
if _, err := tx.ExecContext(ctx, fmt.Sprintf("UPDATE %s SET %s = ? WHERE %s = ?", r.table, r.col, r.col), newID, oldID); err != nil {
return fmt.Errorf("update %s.%s: %w", r.table, r.col, err)
}
}
_, err = tx.ExecContext(ctx, `
INSERT INTO events (issue_id, event_type, actor, old_value, new_value)
VALUES (?, 'renamed', ?, ?, ?)
`, newID, actor, oldID, newID)
return err
}
func updateWispIDInTx(ctx context.Context, tx *sql.Tx, oldID, newID string, issue *types.Issue, actor string) error {
now := time.Now().UTC()
result, err := tx.ExecContext(ctx, `
UPDATE wisps
SET id = ?, title = ?, description = ?, design = ?, acceptance_criteria = ?, notes = ?, updated_at = ?
WHERE id = ?
`, newID, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes, now, oldID)
if err != nil {
return fmt.Errorf("update wisp ID: %w", err)
}
if rows, _ := result.RowsAffected(); rows == 0 {
return fmt.Errorf("wisp not found: %s", oldID)
}
refs := []struct{ table, col string }{
{"wisp_dependencies", "issue_id"},
{"wisp_dependencies", "depends_on_id"},
{"wisp_events", "issue_id"},
{"wisp_labels", "issue_id"},
{"wisp_comments", "issue_id"},
}
for _, r := range refs {
if _, err := tx.ExecContext(ctx, fmt.Sprintf("UPDATE %s SET %s = ? WHERE %s = ?", r.table, r.col, r.col), newID, oldID); err != nil {
return fmt.Errorf("update %s.%s: %w", r.table, r.col, err)
}
}
_, err = tx.ExecContext(ctx, `
INSERT INTO wisp_events (issue_id, event_type, actor, old_value, new_value)
VALUES (?, 'renamed', ?, ?, ?)
`, newID, actor, oldID, newID)
return err
}
// RenameDependencyPrefixInTx updates the prefix in all dependency records.
func RenameDependencyPrefixInTx(ctx context.Context, tx *sql.Tx, oldPrefix, newPrefix string) error {
_, err := tx.ExecContext(ctx, `
UPDATE dependencies
SET issue_id = CONCAT(?, SUBSTRING(issue_id, LENGTH(?) + 1))
WHERE issue_id LIKE CONCAT(?, '%')
`, newPrefix, oldPrefix, oldPrefix)
if err != nil {
return fmt.Errorf("update issue_id prefix in dependencies: %w", err)
}
_, err = tx.ExecContext(ctx, `
UPDATE dependencies
SET depends_on_id = CONCAT(?, SUBSTRING(depends_on_id, LENGTH(?) + 1))
WHERE depends_on_id LIKE CONCAT(?, '%')
`, newPrefix, oldPrefix, oldPrefix)
if err != nil {
return fmt.Errorf("update depends_on_id prefix in dependencies: %w", err)
}
return nil
}
// FindWispDependentsRecursiveInTx walks wisp_dependencies to find all transitive
// dependents of the given IDs.
func FindWispDependentsRecursiveInTx(ctx context.Context, tx *sql.Tx, ids []string) (map[string]bool, error) {
if len(ids) == 0 {
return nil, nil
}
const maxResults = 10000
const batchSize = 50
seen := make(map[string]bool, len(ids))
for _, id := range ids {
seen[id] = true
}
toProcess := make([]string, len(ids))
copy(toProcess, ids)
discovered := make(map[string]bool)
for len(toProcess) > 0 {
if len(seen) > maxResults {
return discovered, fmt.Errorf("wisp cascade traversal discovered over %d issues; aborting", maxResults)
}
end := batchSize
if end > len(toProcess) {
end = len(toProcess)
}
batch := toProcess[:end]
toProcess = toProcess[end:]
placeholders, args := buildSQLInClause(batch)
rows, err := tx.QueryContext(ctx,
fmt.Sprintf(`SELECT issue_id FROM wisp_dependencies WHERE depends_on_id IN (%s)`, placeholders),
args...)
if err != nil {
return discovered, fmt.Errorf("query wisp dependents: %w", err)
}
for rows.Next() {
var depID string
if err := rows.Scan(&depID); err != nil {
_ = rows.Close()
return discovered, fmt.Errorf("scan wisp dependent: %w", err)
}
if !seen[depID] {
seen[depID] = true
discovered[depID] = true
toProcess = append(toProcess, depID)
}
}
_ = rows.Close()
if err := rows.Err(); err != nil {
return discovered, fmt.Errorf("iterate wisp dependents: %w", err)
}
}
return discovered, nil
}
// GetRepoMtimeInTx returns the cached mtime (nanoseconds) for a repo path.
// Returns 0 if no cache entry exists.
func GetRepoMtimeInTx(ctx context.Context, tx *sql.Tx, repoPath string) (int64, error) {
var mtimeNs int64
err := tx.QueryRowContext(ctx,
`SELECT mtime_ns FROM repo_mtimes WHERE repo_path = ?`, repoPath).Scan(&mtimeNs)
if err != nil {
return 0, nil
}
return mtimeNs, nil
}
// SetRepoMtimeInTx upserts the mtime cache for a repo path.
func SetRepoMtimeInTx(ctx context.Context, tx *sql.Tx, repoPath, jsonlPath string, mtimeNs int64) error {
_, err := tx.ExecContext(ctx, `
INSERT INTO repo_mtimes (repo_path, jsonl_path, mtime_ns, last_checked)
VALUES (?, ?, ?, NOW())
ON DUPLICATE KEY UPDATE
jsonl_path = VALUES(jsonl_path),
mtime_ns = VALUES(mtime_ns),
last_checked = NOW()
`, repoPath, jsonlPath, mtimeNs)
if err != nil {
return fmt.Errorf("set repo mtime: %w", err)
}
return nil
}
// ClearRepoMtimeInTx removes the mtime cache entry for a repo path.
func ClearRepoMtimeInTx(ctx context.Context, tx *sql.Tx, repoPath string) error {
// Expand ~ and resolve to absolute path to match stored format.
absPath := expandAndAbsPath(repoPath)
_, err := tx.ExecContext(ctx, `DELETE FROM repo_mtimes WHERE repo_path = ?`, absPath)
if err != nil {
return fmt.Errorf("clear repo mtime: %w", err)
}
return nil
}
func expandAndAbsPath(p string) string {
if strings.HasPrefix(p, "~") {
home, err := os.UserHomeDir()
if err == nil {
if p == "~" {
p = home
} else {
p = filepath.Join(home, p[2:])
}
}
}
abs, err := filepath.Abs(p)
if err != nil {
return p
}
return abs
}