diff --git a/cmd/bd/find_duplicates.go b/cmd/bd/find_duplicates.go new file mode 100644 index 000000000..4d2194847 --- /dev/null +++ b/cmd/bd/find_duplicates.go @@ -0,0 +1,497 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "os" + "sort" + "strings" + "unicode" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/anthropics/anthropic-sdk-go/option" + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" +) + +var findDuplicatesCmd = &cobra.Command{ + Use: "find-duplicates", + Aliases: []string{"find-dups"}, + GroupID: "views", + Short: "Find semantically similar issues using text analysis or AI", + Long: `Find issues that are semantically similar but not exact duplicates. + +Unlike 'bd duplicates' which finds exact content matches, find-duplicates +uses text similarity or AI to find issues that discuss the same topic +with different wording. + +Approaches: + mechanical Token-based text similarity (default, no API key needed) + ai LLM-based semantic comparison (requires ANTHROPIC_API_KEY) + +The mechanical approach tokenizes titles and descriptions, then computes +Jaccard similarity between all issue pairs. It's fast and free but may +miss semantically similar issues with very different wording. + +The AI approach sends candidate pairs to Claude for semantic comparison. +It first uses mechanical pre-filtering to reduce the number of API calls, +then asks the LLM to judge whether the remaining pairs are true duplicates. + +Examples: + bd find-duplicates # Mechanical similarity (default) + bd find-duplicates --threshold 0.4 # Lower threshold = more results + bd find-duplicates --method ai # Use AI for semantic comparison + bd find-duplicates --status open # Only check open issues + bd find-duplicates --limit 20 # Show top 20 pairs + bd find-duplicates --json # JSON output`, + Run: runFindDuplicates, +} + +func init() { + findDuplicatesCmd.Flags().String("method", "mechanical", "Detection method: mechanical, ai") + findDuplicatesCmd.Flags().Float64("threshold", 0.5, "Similarity threshold (0.0-1.0, lower = more results)") + findDuplicatesCmd.Flags().StringP("status", "s", "", "Filter by status (default: non-closed)") + findDuplicatesCmd.Flags().IntP("limit", "n", 50, "Maximum number of pairs to show") + findDuplicatesCmd.Flags().String("model", "claude-haiku-4-5-20251001", "AI model to use (only with --method ai)") + rootCmd.AddCommand(findDuplicatesCmd) +} + +// duplicatePair represents a pair of potentially duplicate issues. +type duplicatePair struct { + IssueA *types.Issue `json:"issue_a"` + IssueB *types.Issue `json:"issue_b"` + Similarity float64 `json:"similarity"` + Method string `json:"method"` + Reason string `json:"reason,omitempty"` +} + +func runFindDuplicates(cmd *cobra.Command, _ []string) { + method, _ := cmd.Flags().GetString("method") + threshold, _ := cmd.Flags().GetFloat64("threshold") + status, _ := cmd.Flags().GetString("status") + limit, _ := cmd.Flags().GetInt("limit") + model, _ := cmd.Flags().GetString("model") + + ctx := rootCtx + + // Validate method + if method != "mechanical" && method != "ai" { + fmt.Fprintf(os.Stderr, "Error: invalid method %q (use: mechanical, ai)\n", method) + os.Exit(1) + } + + // AI method requires API key + if method == "ai" { + if os.Getenv("ANTHROPIC_API_KEY") == "" { + fmt.Fprintf(os.Stderr, "Error: --method ai requires ANTHROPIC_API_KEY environment variable\n") + os.Exit(1) + } + } + + // Check database freshness + if daemonClient == nil { + if err := ensureDatabaseFresh(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + } + + // Fetch issues + filter := types.IssueFilter{} + if status != "" && status != "all" { + s := types.Status(status) + filter.Status = &s + } + + var issues []*types.Issue + var err error + + if daemonClient != nil { + fmt.Fprintf(os.Stderr, "Error: find-duplicates not yet supported in daemon mode\n") + fmt.Fprintf(os.Stderr, "Use: bd --no-daemon find-duplicates\n") + os.Exit(1) + } + + issues, err = store.SearchIssues(ctx, "", filter) + if err != nil { + fmt.Fprintf(os.Stderr, "Error fetching issues: %v\n", err) + os.Exit(1) + } + + // Default: filter out closed issues unless status flag is set + if status == "" { + var filtered []*types.Issue + for _, issue := range issues { + if issue.Status != types.StatusClosed { + filtered = append(filtered, issue) + } + } + issues = filtered + } + + if len(issues) < 2 { + if jsonOutput { + outputJSON(map[string]interface{}{ + "pairs": []interface{}{}, + "count": 0, + }) + } else { + fmt.Println("Not enough issues to compare (need at least 2)") + } + return + } + + // Find duplicate pairs + var pairs []duplicatePair + switch method { + case "mechanical": + pairs = findMechanicalDuplicates(issues, threshold) + case "ai": + pairs = findAIDuplicates(ctx, issues, threshold, model) + } + + // Sort by similarity (highest first) + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Similarity > pairs[j].Similarity + }) + + // Apply limit + if limit > 0 && len(pairs) > limit { + pairs = pairs[:limit] + } + + // Output + if jsonOutput { + type pairJSON struct { + IssueAID string `json:"issue_a_id"` + IssueBID string `json:"issue_b_id"` + IssueATitle string `json:"issue_a_title"` + IssueBTitle string `json:"issue_b_title"` + Similarity float64 `json:"similarity"` + Method string `json:"method"` + Reason string `json:"reason,omitempty"` + } + jsonPairs := make([]pairJSON, len(pairs)) + for i, p := range pairs { + jsonPairs[i] = pairJSON{ + IssueAID: p.IssueA.ID, + IssueBID: p.IssueB.ID, + IssueATitle: p.IssueA.Title, + IssueBTitle: p.IssueB.Title, + Similarity: p.Similarity, + Method: p.Method, + Reason: p.Reason, + } + } + outputJSON(map[string]interface{}{ + "pairs": jsonPairs, + "count": len(jsonPairs), + "method": method, + "threshold": threshold, + }) + return + } + + if len(pairs) == 0 { + fmt.Printf("No similar issues found (threshold: %.0f%%)\n", threshold*100) + return + } + + fmt.Printf("%s Found %d potential duplicate pair(s) (threshold: %.0f%%):\n\n", + ui.RenderWarn("🔍"), len(pairs), threshold*100) + + for i, p := range pairs { + pct := p.Similarity * 100 + fmt.Printf("%s Pair %d (%.0f%% similar):\n", ui.RenderAccent("━━"), i+1, pct) + fmt.Printf(" %s %s\n", ui.RenderPass(p.IssueA.ID), p.IssueA.Title) + fmt.Printf(" %s %s\n", ui.RenderPass(p.IssueB.ID), p.IssueB.Title) + if p.Reason != "" { + fmt.Printf(" %s %s\n", ui.RenderAccent("Reason:"), p.Reason) + } + fmt.Printf(" %s bd show %s %s\n\n", ui.RenderAccent("Compare:"), p.IssueA.ID, p.IssueB.ID) + } +} + +// tokenize splits text into lowercase word tokens, removing punctuation. +func tokenize(text string) map[string]int { + tokens := make(map[string]int) + words := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' + }) + for _, w := range words { + if len(w) > 1 { // Skip single chars + tokens[w]++ + } + } + return tokens +} + +// issueText returns the combined text content of an issue for comparison. +func issueText(issue *types.Issue) string { + parts := []string{issue.Title} + if issue.Description != "" { + parts = append(parts, issue.Description) + } + return strings.Join(parts, " ") +} + +// jaccardSimilarity computes the Jaccard similarity between two token sets. +func jaccardSimilarity(a, b map[string]int) float64 { + if len(a) == 0 && len(b) == 0 { + return 0 + } + + intersection := 0 + union := 0 + + // Count union from a + for token, countA := range a { + if countB, ok := b[token]; ok { + if countA < countB { + intersection += countA + } else { + intersection += countB + } + if countA > countB { + union += countA + } else { + union += countB + } + } else { + union += countA + } + } + // Count tokens only in b + for token, countB := range b { + if _, ok := a[token]; !ok { + union += countB + } + } + + if union == 0 { + return 0 + } + return float64(intersection) / float64(union) +} + +// cosineSimilarity computes the cosine similarity between two token vectors. +func cosineSimilarity(a, b map[string]int) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + + dotProduct := 0.0 + magA := 0.0 + magB := 0.0 + + for token, countA := range a { + fa := float64(countA) + magA += fa * fa + if countB, ok := b[token]; ok { + dotProduct += fa * float64(countB) + } + } + for _, countB := range b { + fb := float64(countB) + magB += fb * fb + } + + if magA == 0 || magB == 0 { + return 0 + } + return dotProduct / (math.Sqrt(magA) * math.Sqrt(magB)) +} + +// findMechanicalDuplicates finds similar issues using token-based text similarity. +func findMechanicalDuplicates(issues []*types.Issue, threshold float64) []duplicatePair { + // Pre-tokenize all issues + type tokenized struct { + issue *types.Issue + tokens map[string]int + } + items := make([]tokenized, len(issues)) + for i, issue := range issues { + items[i] = tokenized{ + issue: issue, + tokens: tokenize(issueText(issue)), + } + } + + var pairs []duplicatePair + + // Compare all pairs + for i := 0; i < len(items); i++ { + for j := i + 1; j < len(items); j++ { + // Use average of Jaccard and cosine for better accuracy + jaccard := jaccardSimilarity(items[i].tokens, items[j].tokens) + cosine := cosineSimilarity(items[i].tokens, items[j].tokens) + similarity := (jaccard + cosine) / 2 + + if similarity >= threshold { + pairs = append(pairs, duplicatePair{ + IssueA: items[i].issue, + IssueB: items[j].issue, + Similarity: similarity, + Method: "mechanical", + }) + } + } + } + + return pairs +} + +// findAIDuplicates uses LLM-based semantic comparison to find duplicates. +// It first pre-filters with mechanical similarity to reduce API calls. +func findAIDuplicates(ctx context.Context, issues []*types.Issue, threshold float64, model string) []duplicatePair { + // Pre-filter: use mechanical similarity with a lower threshold to find candidates + preFilterThreshold := threshold * 0.5 // Cast a wider net for pre-filtering + if preFilterThreshold < 0.15 { + preFilterThreshold = 0.15 + } + candidates := findMechanicalDuplicates(issues, preFilterThreshold) + + if len(candidates) == 0 { + return nil + } + + // Limit candidates to avoid excessive API calls + maxCandidates := 100 + if len(candidates) > maxCandidates { + // Sort by mechanical similarity and take the top candidates + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].Similarity > candidates[j].Similarity + }) + candidates = candidates[:maxCandidates] + } + + fmt.Fprintf(os.Stderr, "Analyzing %d candidate pairs with AI...\n", len(candidates)) + + apiKey := os.Getenv("ANTHROPIC_API_KEY") + client := anthropic.NewClient(option.WithAPIKey(apiKey)) + + var pairs []duplicatePair + + // Batch candidates into groups for efficient API usage + batchSize := 10 + for i := 0; i < len(candidates); i += batchSize { + end := i + batchSize + if end > len(candidates) { + end = len(candidates) + } + batch := candidates[i:end] + + results := analyzeWithAI(ctx, client, anthropic.Model(model), batch) + for _, r := range results { + if r.Similarity >= threshold { + pairs = append(pairs, r) + } + } + } + + return pairs +} + +// analyzeWithAI sends a batch of candidate pairs to the LLM for semantic comparison. +func analyzeWithAI(ctx context.Context, client anthropic.Client, model anthropic.Model, candidates []duplicatePair) []duplicatePair { + if len(candidates) == 0 { + return nil + } + + // Build the prompt + var sb strings.Builder + sb.WriteString("You are analyzing issue pairs to determine if they are semantic duplicates.\n") + sb.WriteString("For each pair, determine if they describe the same problem/task/feature.\n") + sb.WriteString("Respond with a JSON array of objects, one per pair, with fields:\n") + sb.WriteString(" - pair_index (int): 0-based index of the pair\n") + sb.WriteString(" - is_duplicate (bool): true if semantically the same issue\n") + sb.WriteString(" - confidence (float): 0.0-1.0 how confident you are\n") + sb.WriteString(" - reason (string): brief explanation\n\n") + sb.WriteString("Respond ONLY with the JSON array, no other text.\n\n") + + for i, c := range candidates { + fmt.Fprintf(&sb, "--- Pair %d ---\n", i) + fmt.Fprintf(&sb, "Issue A [%s]: %s\n", c.IssueA.ID, c.IssueA.Title) + if c.IssueA.Description != "" { + descA := c.IssueA.Description + if len(descA) > 500 { + descA = descA[:500] + "..." + } + fmt.Fprintf(&sb, " Description: %s\n", descA) + } + fmt.Fprintf(&sb, "Issue B [%s]: %s\n", c.IssueB.ID, c.IssueB.Title) + if c.IssueB.Description != "" { + descB := c.IssueB.Description + if len(descB) > 500 { + descB = descB[:500] + "..." + } + fmt.Fprintf(&sb, " Description: %s\n", descB) + } + sb.WriteString("\n") + } + + message, err := client.Messages.New(ctx, anthropic.MessageNewParams{ + Model: model, + MaxTokens: 2048, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock(sb.String())), + }, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: AI analysis failed: %v\n", err) + // Fall back to mechanical scores + return candidates + } + + if len(message.Content) == 0 || message.Content[0].Type != "text" { + fmt.Fprintf(os.Stderr, "Warning: unexpected AI response format\n") + return candidates + } + + // Parse the JSON response + responseText := message.Content[0].Text + + // Try to extract JSON from the response (handle markdown code blocks) + jsonText := responseText + if idx := strings.Index(jsonText, "["); idx >= 0 { + jsonText = jsonText[idx:] + } + if idx := strings.LastIndex(jsonText, "]"); idx >= 0 { + jsonText = jsonText[:idx+1] + } + + var results []struct { + PairIndex int `json:"pair_index"` + IsDuplicate bool `json:"is_duplicate"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason"` + } + + if err := json.Unmarshal([]byte(jsonText), &results); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to parse AI response: %v\n", err) + return candidates + } + + // Map results back to candidate pairs + var pairs []duplicatePair + for _, r := range results { + if r.PairIndex < 0 || r.PairIndex >= len(candidates) { + continue + } + if r.IsDuplicate { + c := candidates[r.PairIndex] + pairs = append(pairs, duplicatePair{ + IssueA: c.IssueA, + IssueB: c.IssueB, + Similarity: r.Confidence, + Method: "ai", + Reason: r.Reason, + }) + } + } + + return pairs +} diff --git a/cmd/bd/find_duplicates_test.go b/cmd/bd/find_duplicates_test.go new file mode 100644 index 000000000..ff3aad105 --- /dev/null +++ b/cmd/bd/find_duplicates_test.go @@ -0,0 +1,263 @@ +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +func TestTokenize(t *testing.T) { + tests := []struct { + name string + input string + expected map[string]int + }{ + { + name: "simple words", + input: "fix authentication bug", + expected: map[string]int{ + "fix": 1, + "authentication": 1, + "bug": 1, + }, + }, + { + name: "duplicate words", + input: "fix the bug fix", + expected: map[string]int{ + "fix": 2, + "the": 1, + "bug": 1, + }, + }, + { + name: "mixed case and punctuation", + input: "Fix: Authentication BUG!", + expected: map[string]int{ + "fix": 1, + "authentication": 1, + "bug": 1, + }, + }, + { + name: "hyphenated words preserved", + input: "auto-import feature", + expected: map[string]int{ + "auto-import": 1, + "feature": 1, + }, + }, + { + name: "single chars filtered", + input: "a b c hello", + expected: map[string]int{"hello": 1}, + }, + { + name: "empty string", + input: "", + expected: map[string]int{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tokenize(tt.input) + if len(result) != len(tt.expected) { + t.Errorf("tokenize(%q) got %d tokens, want %d: %v", tt.input, len(result), len(tt.expected), result) + return + } + for k, v := range tt.expected { + if result[k] != v { + t.Errorf("tokenize(%q)[%q] = %d, want %d", tt.input, k, result[k], v) + } + } + }) + } +} + +func TestJaccardSimilarity(t *testing.T) { + tests := []struct { + name string + a, b map[string]int + expected float64 + epsilon float64 + }{ + { + name: "identical sets", + a: map[string]int{"fix": 1, "bug": 1}, + b: map[string]int{"fix": 1, "bug": 1}, + expected: 1.0, + epsilon: 0.001, + }, + { + name: "disjoint sets", + a: map[string]int{"fix": 1, "bug": 1}, + b: map[string]int{"add": 1, "feature": 1}, + expected: 0.0, + epsilon: 0.001, + }, + { + name: "partial overlap", + a: map[string]int{"fix": 1, "bug": 1, "auth": 1}, + b: map[string]int{"fix": 1, "auth": 1, "login": 1}, + expected: 0.5, // intersection=2, union=4 + epsilon: 0.001, + }, + { + name: "both empty", + a: map[string]int{}, + b: map[string]int{}, + expected: 0.0, + epsilon: 0.001, + }, + { + name: "one empty", + a: map[string]int{"fix": 1}, + b: map[string]int{}, + expected: 0.0, + epsilon: 0.001, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := jaccardSimilarity(tt.a, tt.b) + diff := result - tt.expected + if diff < 0 { + diff = -diff + } + if diff > tt.epsilon { + t.Errorf("jaccardSimilarity() = %f, want %f (diff %f)", result, tt.expected, diff) + } + }) + } +} + +func TestCosineSimilarity(t *testing.T) { + tests := []struct { + name string + a, b map[string]int + expected float64 + epsilon float64 + }{ + { + name: "identical", + a: map[string]int{"fix": 1, "bug": 1}, + b: map[string]int{"fix": 1, "bug": 1}, + expected: 1.0, + epsilon: 0.001, + }, + { + name: "orthogonal", + a: map[string]int{"fix": 1}, + b: map[string]int{"add": 1}, + expected: 0.0, + epsilon: 0.001, + }, + { + name: "empty sets", + a: map[string]int{}, + b: map[string]int{}, + expected: 0.0, + epsilon: 0.001, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := cosineSimilarity(tt.a, tt.b) + diff := result - tt.expected + if diff < 0 { + diff = -diff + } + if diff > tt.epsilon { + t.Errorf("cosineSimilarity() = %f, want %f", result, tt.expected) + } + }) + } +} + +func TestFindMechanicalDuplicates(t *testing.T) { + issues := []*types.Issue{ + { + ID: "bd-001", + Title: "Fix authentication bug in login flow", + Description: "The login page fails when using SSO authentication", + }, + { + ID: "bd-002", + Title: "Authentication login bug fix", + Description: "SSO authentication fails on the login page", + }, + { + ID: "bd-003", + Title: "Add dark mode support", + Description: "Implement dark mode theme for the application", + }, + } + + pairs := findMechanicalDuplicates(issues, 0.3) + + // Should find bd-001 and bd-002 as similar + found := false + for _, p := range pairs { + if (p.IssueA.ID == "bd-001" && p.IssueB.ID == "bd-002") || + (p.IssueA.ID == "bd-002" && p.IssueB.ID == "bd-001") { + found = true + if p.Similarity < 0.3 { + t.Errorf("expected similarity >= 0.3 for auth bug pair, got %f", p.Similarity) + } + } + } + if !found { + t.Error("expected to find auth bug pair as similar") + } + + // bd-003 should NOT match with bd-001 or bd-002 + for _, p := range pairs { + if p.IssueA.ID == "bd-003" || p.IssueB.ID == "bd-003" { + t.Errorf("dark mode issue should not match auth issues, got similarity %f", p.Similarity) + } + } +} + +func TestFindMechanicalDuplicatesHighThreshold(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-001", Title: "Fix bug A", Description: "Something completely different"}, + {ID: "bd-002", Title: "Add feature B", Description: "Totally unrelated content"}, + } + + pairs := findMechanicalDuplicates(issues, 0.9) + if len(pairs) != 0 { + t.Errorf("expected no pairs at threshold 0.9 for different issues, got %d", len(pairs)) + } +} + +func TestFindMechanicalDuplicatesMinIssues(t *testing.T) { + // Single issue - no pairs possible + issues := []*types.Issue{ + {ID: "bd-001", Title: "Only issue"}, + } + pairs := findMechanicalDuplicates(issues, 0.1) + if len(pairs) != 0 { + t.Errorf("expected no pairs for single issue, got %d", len(pairs)) + } +} + +func TestIssueText(t *testing.T) { + issue := &types.Issue{ + Title: "Fix bug", + Description: "detailed description", + } + text := issueText(issue) + if text != "Fix bug detailed description" { + t.Errorf("issueText() = %q, want %q", text, "Fix bug detailed description") + } + + // No description + issue2 := &types.Issue{Title: "Just title"} + text2 := issueText(issue2) + if text2 != "Just title" { + t.Errorf("issueText() = %q, want %q", text2, "Just title") + } +}