feat: add bd find-duplicates for AI-powered duplicate detection (bd-581b80b3)

Adds semantic duplicate detection that goes beyond exact content matching.
Two methods: mechanical (token-based Jaccard+cosine similarity, default)
and AI (LLM-based via Claude API with mechanical pre-filtering).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
beads/crew/lydia 2026-02-06 10:33:32 -08:00 • committed by Steve Yegge
commit e06e0cc866
2 changed files with 760 additions and 0 deletions

497
cmd/bd/find_duplicates.go Normal file
View file

@ -0,0 +1,497 @@
package main
import (
"context"
"encoding/json"
"fmt"
"math"
"os"
"sort"
"strings"
"unicode"
"github.com/anthropics/anthropic-sdk-go"
"github.com/anthropics/anthropic-sdk-go/option"
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/types"
"github.com/steveyegge/beads/internal/ui"
)
var findDuplicatesCmd = &cobra.Command{
Use: "find-duplicates",
Aliases: []string{"find-dups"},
GroupID: "views",
Short: "Find semantically similar issues using text analysis or AI",
Long: `Find issues that are semantically similar but not exact duplicates.
Unlike 'bd duplicates' which finds exact content matches, find-duplicates
uses text similarity or AI to find issues that discuss the same topic
with different wording.
Approaches:
mechanical Token-based text similarity (default, no API key needed)
ai LLM-based semantic comparison (requires ANTHROPIC_API_KEY)
The mechanical approach tokenizes titles and descriptions, then computes
Jaccard similarity between all issue pairs. It's fast and free but may
miss semantically similar issues with very different wording.
The AI approach sends candidate pairs to Claude for semantic comparison.
It first uses mechanical pre-filtering to reduce the number of API calls,
then asks the LLM to judge whether the remaining pairs are true duplicates.
Examples:
bd find-duplicates # Mechanical similarity (default)
bd find-duplicates --threshold 0.4 # Lower threshold = more results
bd find-duplicates --method ai # Use AI for semantic comparison
bd find-duplicates --status open # Only check open issues
bd find-duplicates --limit 20 # Show top 20 pairs
bd find-duplicates --json # JSON output`,
Run: runFindDuplicates,
}
func init() {
findDuplicatesCmd.Flags().String("method", "mechanical", "Detection method: mechanical, ai")
findDuplicatesCmd.Flags().Float64("threshold", 0.5, "Similarity threshold (0.0-1.0, lower = more results)")
findDuplicatesCmd.Flags().StringP("status", "s", "", "Filter by status (default: non-closed)")
findDuplicatesCmd.Flags().IntP("limit", "n", 50, "Maximum number of pairs to show")
findDuplicatesCmd.Flags().String("model", "claude-haiku-4-5-20251001", "AI model to use (only with --method ai)")
rootCmd.AddCommand(findDuplicatesCmd)
}
// duplicatePair represents a pair of potentially duplicate issues.
type duplicatePair struct {
IssueA *types.Issue `json:"issue_a"`
IssueB *types.Issue `json:"issue_b"`
Similarity float64 `json:"similarity"`
Method string `json:"method"`
Reason string `json:"reason,omitempty"`
}
func runFindDuplicates(cmd *cobra.Command, _ []string) {
method, _ := cmd.Flags().GetString("method")
threshold, _ := cmd.Flags().GetFloat64("threshold")
status, _ := cmd.Flags().GetString("status")
limit, _ := cmd.Flags().GetInt("limit")
model, _ := cmd.Flags().GetString("model")
ctx := rootCtx
// Validate method
if method != "mechanical" && method != "ai" {
fmt.Fprintf(os.Stderr, "Error: invalid method %q (use: mechanical, ai)\n", method)
os.Exit(1)
}
// AI method requires API key
if method == "ai" {
if os.Getenv("ANTHROPIC_API_KEY") == "" {
fmt.Fprintf(os.Stderr, "Error: --method ai requires ANTHROPIC_API_KEY environment variable\n")
os.Exit(1)
}
}
// Check database freshness
if daemonClient == nil {
if err := ensureDatabaseFresh(ctx); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
// Fetch issues
filter := types.IssueFilter{}
if status != "" && status != "all" {
s := types.Status(status)
filter.Status = &s
}
var issues []*types.Issue
var err error
if daemonClient != nil {
fmt.Fprintf(os.Stderr, "Error: find-duplicates not yet supported in daemon mode\n")
fmt.Fprintf(os.Stderr, "Use: bd --no-daemon find-duplicates\n")
os.Exit(1)
}
issues, err = store.SearchIssues(ctx, "", filter)
if err != nil {
fmt.Fprintf(os.Stderr, "Error fetching issues: %v\n", err)
os.Exit(1)
}
// Default: filter out closed issues unless status flag is set
if status == "" {
var filtered []*types.Issue
for _, issue := range issues {
if issue.Status != types.StatusClosed {
filtered = append(filtered, issue)
}
}
issues = filtered
}
if len(issues) < 2 {
if jsonOutput {
outputJSON(map[string]interface{}{
"pairs": []interface{}{},
"count": 0,
})
} else {
fmt.Println("Not enough issues to compare (need at least 2)")
}
return
}
// Find duplicate pairs
var pairs []duplicatePair
switch method {
case "mechanical":
pairs = findMechanicalDuplicates(issues, threshold)
case "ai":
pairs = findAIDuplicates(ctx, issues, threshold, model)
}
// Sort by similarity (highest first)
sort.Slice(pairs, func(i, j int) bool {
return pairs[i].Similarity > pairs[j].Similarity
})
// Apply limit
if limit > 0 && len(pairs) > limit {
pairs = pairs[:limit]
}
// Output
if jsonOutput {
type pairJSON struct {
IssueAID string `json:"issue_a_id"`
IssueBID string `json:"issue_b_id"`
IssueATitle string `json:"issue_a_title"`
IssueBTitle string `json:"issue_b_title"`
Similarity float64 `json:"similarity"`
Method string `json:"method"`
Reason string `json:"reason,omitempty"`
}
jsonPairs := make([]pairJSON, len(pairs))
for i, p := range pairs {
jsonPairs[i] = pairJSON{
IssueAID: p.IssueA.ID,
IssueBID: p.IssueB.ID,
IssueATitle: p.IssueA.Title,
IssueBTitle: p.IssueB.Title,
Similarity: p.Similarity,
Method: p.Method,
Reason: p.Reason,
}
}
outputJSON(map[string]interface{}{
"pairs": jsonPairs,
"count": len(jsonPairs),
"method": method,
"threshold": threshold,
})
return
}
if len(pairs) == 0 {
fmt.Printf("No similar issues found (threshold: %.0f%%)\n", threshold*100)
return
}
fmt.Printf("%s Found %d potential duplicate pair(s) (threshold: %.0f%%):\n\n",
ui.RenderWarn("🔍"), len(pairs), threshold*100)
for i, p := range pairs {
pct := p.Similarity * 100
fmt.Printf("%s Pair %d (%.0f%% similar):\n", ui.RenderAccent("━━"), i+1, pct)
fmt.Printf(" %s %s\n", ui.RenderPass(p.IssueA.ID), p.IssueA.Title)
fmt.Printf(" %s %s\n", ui.RenderPass(p.IssueB.ID), p.IssueB.Title)
if p.Reason != "" {
fmt.Printf(" %s %s\n", ui.RenderAccent("Reason:"), p.Reason)
}
fmt.Printf(" %s bd show %s %s\n\n", ui.RenderAccent("Compare:"), p.IssueA.ID, p.IssueB.ID)
}
}
// tokenize splits text into lowercase word tokens, removing punctuation.
func tokenize(text string) map[string]int {
tokens := make(map[string]int)
words := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-'
})
for _, w := range words {
if len(w) > 1 { // Skip single chars
tokens[w]++
}
}
return tokens
}
// issueText returns the combined text content of an issue for comparison.
func issueText(issue *types.Issue) string {
parts := []string{issue.Title}
if issue.Description != "" {
parts = append(parts, issue.Description)
}
return strings.Join(parts, " ")
}
// jaccardSimilarity computes the Jaccard similarity between two token sets.
func jaccardSimilarity(a, b map[string]int) float64 {
if len(a) == 0 && len(b) == 0 {
return 0
}
intersection := 0
union := 0
// Count union from a
for token, countA := range a {
if countB, ok := b[token]; ok {
if countA < countB {
intersection += countA
} else {
intersection += countB
}
if countA > countB {
union += countA
} else {
union += countB
}
} else {
union += countA
}
}
// Count tokens only in b
for token, countB := range b {
if _, ok := a[token]; !ok {
union += countB
}
}
if union == 0 {
return 0
}
return float64(intersection) / float64(union)
}
// cosineSimilarity computes the cosine similarity between two token vectors.
func cosineSimilarity(a, b map[string]int) float64 {
if len(a) == 0 || len(b) == 0 {
return 0
}
dotProduct := 0.0
magA := 0.0
magB := 0.0
for token, countA := range a {
fa := float64(countA)
magA += fa * fa
if countB, ok := b[token]; ok {
dotProduct += fa * float64(countB)
}
}
for _, countB := range b {
fb := float64(countB)
magB += fb * fb
}
if magA == 0 || magB == 0 {
return 0
}
return dotProduct / (math.Sqrt(magA) * math.Sqrt(magB))
}
// findMechanicalDuplicates finds similar issues using token-based text similarity.
func findMechanicalDuplicates(issues []*types.Issue, threshold float64) []duplicatePair {
// Pre-tokenize all issues
type tokenized struct {
issue *types.Issue
tokens map[string]int
}
items := make([]tokenized, len(issues))
for i, issue := range issues {
items[i] = tokenized{
issue: issue,
tokens: tokenize(issueText(issue)),
}
}
var pairs []duplicatePair
// Compare all pairs
for i := 0; i < len(items); i++ {
for j := i + 1; j < len(items); j++ {
// Use average of Jaccard and cosine for better accuracy
jaccard := jaccardSimilarity(items[i].tokens, items[j].tokens)
cosine := cosineSimilarity(items[i].tokens, items[j].tokens)
similarity := (jaccard + cosine) / 2
if similarity >= threshold {
pairs = append(pairs, duplicatePair{
IssueA: items[i].issue,
IssueB: items[j].issue,
Similarity: similarity,
Method: "mechanical",
})
}
}
}
return pairs
}
// findAIDuplicates uses LLM-based semantic comparison to find duplicates.
// It first pre-filters with mechanical similarity to reduce API calls.
func findAIDuplicates(ctx context.Context, issues []*types.Issue, threshold float64, model string) []duplicatePair {
// Pre-filter: use mechanical similarity with a lower threshold to find candidates
preFilterThreshold := threshold * 0.5 // Cast a wider net for pre-filtering
if preFilterThreshold < 0.15 {
preFilterThreshold = 0.15
}
candidates := findMechanicalDuplicates(issues, preFilterThreshold)
if len(candidates) == 0 {
return nil
}
// Limit candidates to avoid excessive API calls
maxCandidates := 100
if len(candidates) > maxCandidates {
// Sort by mechanical similarity and take the top candidates
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].Similarity > candidates[j].Similarity
})
candidates = candidates[:maxCandidates]
}
fmt.Fprintf(os.Stderr, "Analyzing %d candidate pairs with AI...\n", len(candidates))
apiKey := os.Getenv("ANTHROPIC_API_KEY")
client := anthropic.NewClient(option.WithAPIKey(apiKey))
var pairs []duplicatePair
// Batch candidates into groups for efficient API usage
batchSize := 10
for i := 0; i < len(candidates); i += batchSize {
end := i + batchSize
if end > len(candidates) {
end = len(candidates)
}
batch := candidates[i:end]
results := analyzeWithAI(ctx, client, anthropic.Model(model), batch)
for _, r := range results {
if r.Similarity >= threshold {
pairs = append(pairs, r)
}
}
}
return pairs
}
// analyzeWithAI sends a batch of candidate pairs to the LLM for semantic comparison.
func analyzeWithAI(ctx context.Context, client anthropic.Client, model anthropic.Model, candidates []duplicatePair) []duplicatePair {
if len(candidates) == 0 {
return nil
}
// Build the prompt
var sb strings.Builder
sb.WriteString("You are analyzing issue pairs to determine if they are semantic duplicates.\n")
sb.WriteString("For each pair, determine if they describe the same problem/task/feature.\n")
sb.WriteString("Respond with a JSON array of objects, one per pair, with fields:\n")
sb.WriteString(" - pair_index (int): 0-based index of the pair\n")
sb.WriteString(" - is_duplicate (bool): true if semantically the same issue\n")
sb.WriteString(" - confidence (float): 0.0-1.0 how confident you are\n")
sb.WriteString(" - reason (string): brief explanation\n\n")
sb.WriteString("Respond ONLY with the JSON array, no other text.\n\n")
for i, c := range candidates {
fmt.Fprintf(&sb, "--- Pair %d ---\n", i)
fmt.Fprintf(&sb, "Issue A [%s]: %s\n", c.IssueA.ID, c.IssueA.Title)
if c.IssueA.Description != "" {
descA := c.IssueA.Description
if len(descA) > 500 {
descA = descA[:500] + "..."
}
fmt.Fprintf(&sb, " Description: %s\n", descA)
}
fmt.Fprintf(&sb, "Issue B [%s]: %s\n", c.IssueB.ID, c.IssueB.Title)
if c.IssueB.Description != "" {
descB := c.IssueB.Description
if len(descB) > 500 {
descB = descB[:500] + "..."
}
fmt.Fprintf(&sb, " Description: %s\n", descB)
}
sb.WriteString("\n")
}
message, err := client.Messages.New(ctx, anthropic.MessageNewParams{
Model: model,
MaxTokens: 2048,
Messages: []anthropic.MessageParam{
anthropic.NewUserMessage(anthropic.NewTextBlock(sb.String())),
},
})
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: AI analysis failed: %v\n", err)
// Fall back to mechanical scores
return candidates
}
if len(message.Content) == 0 || message.Content[0].Type != "text" {
fmt.Fprintf(os.Stderr, "Warning: unexpected AI response format\n")
return candidates
}
// Parse the JSON response
responseText := message.Content[0].Text
// Try to extract JSON from the response (handle markdown code blocks)
jsonText := responseText
if idx := strings.Index(jsonText, "["); idx >= 0 {
jsonText = jsonText[idx:]
}
if idx := strings.LastIndex(jsonText, "]"); idx >= 0 {
jsonText = jsonText[:idx+1]
}
var results []struct {
PairIndex int `json:"pair_index"`
IsDuplicate bool `json:"is_duplicate"`
Confidence float64 `json:"confidence"`
Reason string `json:"reason"`
}
if err := json.Unmarshal([]byte(jsonText), &results); err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to parse AI response: %v\n", err)
return candidates
}
// Map results back to candidate pairs
var pairs []duplicatePair
for _, r := range results {
if r.PairIndex < 0 || r.PairIndex >= len(candidates) {
continue
}
if r.IsDuplicate {
c := candidates[r.PairIndex]
pairs = append(pairs, duplicatePair{
IssueA: c.IssueA,
IssueB: c.IssueB,
Similarity: r.Confidence,
Method: "ai",
Reason: r.Reason,
})
}
}
return pairs
}

View file

@ -0,0 +1,263 @@
package main
import (
"testing"
"github.com/steveyegge/beads/internal/types"
)
func TestTokenize(t *testing.T) {
tests := []struct {
name string
input string
expected map[string]int
}{
{
name: "simple words",
input: "fix authentication bug",
expected: map[string]int{
"fix": 1,
"authentication": 1,
"bug": 1,
},
},
{
name: "duplicate words",
input: "fix the bug fix",
expected: map[string]int{
"fix": 2,
"the": 1,
"bug": 1,
},
},
{
name: "mixed case and punctuation",
input: "Fix: Authentication BUG!",
expected: map[string]int{
"fix": 1,
"authentication": 1,
"bug": 1,
},
},
{
name: "hyphenated words preserved",
input: "auto-import feature",
expected: map[string]int{
"auto-import": 1,
"feature": 1,
},
},
{
name: "single chars filtered",
input: "a b c hello",
expected: map[string]int{"hello": 1},
},
{
name: "empty string",
input: "",
expected: map[string]int{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tokenize(tt.input)
if len(result) != len(tt.expected) {
t.Errorf("tokenize(%q) got %d tokens, want %d: %v", tt.input, len(result), len(tt.expected), result)
return
}
for k, v := range tt.expected {
if result[k] != v {
t.Errorf("tokenize(%q)[%q] = %d, want %d", tt.input, k, result[k], v)
}
}
})
}
}
func TestJaccardSimilarity(t *testing.T) {
tests := []struct {
name string
a, b map[string]int
expected float64
epsilon float64
}{
{
name: "identical sets",
a: map[string]int{"fix": 1, "bug": 1},
b: map[string]int{"fix": 1, "bug": 1},
expected: 1.0,
epsilon: 0.001,
},
{
name: "disjoint sets",
a: map[string]int{"fix": 1, "bug": 1},
b: map[string]int{"add": 1, "feature": 1},
expected: 0.0,
epsilon: 0.001,
},
{
name: "partial overlap",
a: map[string]int{"fix": 1, "bug": 1, "auth": 1},
b: map[string]int{"fix": 1, "auth": 1, "login": 1},
expected: 0.5, // intersection=2, union=4
epsilon: 0.001,
},
{
name: "both empty",
a: map[string]int{},
b: map[string]int{},
expected: 0.0,
epsilon: 0.001,
},
{
name: "one empty",
a: map[string]int{"fix": 1},
b: map[string]int{},
expected: 0.0,
epsilon: 0.001,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := jaccardSimilarity(tt.a, tt.b)
diff := result - tt.expected
if diff < 0 {
diff = -diff
}
if diff > tt.epsilon {
t.Errorf("jaccardSimilarity() = %f, want %f (diff %f)", result, tt.expected, diff)
}
})
}
}
func TestCosineSimilarity(t *testing.T) {
tests := []struct {
name string
a, b map[string]int
expected float64
epsilon float64
}{
{
name: "identical",
a: map[string]int{"fix": 1, "bug": 1},
b: map[string]int{"fix": 1, "bug": 1},
expected: 1.0,
epsilon: 0.001,
},
{
name: "orthogonal",
a: map[string]int{"fix": 1},
b: map[string]int{"add": 1},
expected: 0.0,
epsilon: 0.001,
},
{
name: "empty sets",
a: map[string]int{},
b: map[string]int{},
expected: 0.0,
epsilon: 0.001,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := cosineSimilarity(tt.a, tt.b)
diff := result - tt.expected
if diff < 0 {
diff = -diff
}
if diff > tt.epsilon {
t.Errorf("cosineSimilarity() = %f, want %f", result, tt.expected)
}
})
}
}
func TestFindMechanicalDuplicates(t *testing.T) {
issues := []*types.Issue{
{
ID: "bd-001",
Title: "Fix authentication bug in login flow",
Description: "The login page fails when using SSO authentication",
},
{
ID: "bd-002",
Title: "Authentication login bug fix",
Description: "SSO authentication fails on the login page",
},
{
ID: "bd-003",
Title: "Add dark mode support",
Description: "Implement dark mode theme for the application",
},
}
pairs := findMechanicalDuplicates(issues, 0.3)
// Should find bd-001 and bd-002 as similar
found := false
for _, p := range pairs {
if (p.IssueA.ID == "bd-001" && p.IssueB.ID == "bd-002") ||
(p.IssueA.ID == "bd-002" && p.IssueB.ID == "bd-001") {
found = true
if p.Similarity < 0.3 {
t.Errorf("expected similarity >= 0.3 for auth bug pair, got %f", p.Similarity)
}
}
}
if !found {
t.Error("expected to find auth bug pair as similar")
}
// bd-003 should NOT match with bd-001 or bd-002
for _, p := range pairs {
if p.IssueA.ID == "bd-003" || p.IssueB.ID == "bd-003" {
t.Errorf("dark mode issue should not match auth issues, got similarity %f", p.Similarity)
}
}
}
func TestFindMechanicalDuplicatesHighThreshold(t *testing.T) {
issues := []*types.Issue{
{ID: "bd-001", Title: "Fix bug A", Description: "Something completely different"},
{ID: "bd-002", Title: "Add feature B", Description: "Totally unrelated content"},
}
pairs := findMechanicalDuplicates(issues, 0.9)
if len(pairs) != 0 {
t.Errorf("expected no pairs at threshold 0.9 for different issues, got %d", len(pairs))
}
}
func TestFindMechanicalDuplicatesMinIssues(t *testing.T) {
// Single issue - no pairs possible
issues := []*types.Issue{
{ID: "bd-001", Title: "Only issue"},
}
pairs := findMechanicalDuplicates(issues, 0.1)
if len(pairs) != 0 {
t.Errorf("expected no pairs for single issue, got %d", len(pairs))
}
}
func TestIssueText(t *testing.T) {
issue := &types.Issue{
Title: "Fix bug",
Description: "detailed description",
}
text := issueText(issue)
if text != "Fix bug detailed description" {
t.Errorf("issueText() = %q, want %q", text, "Fix bug detailed description")
}
// No description
issue2 := &types.Issue{Title: "Just title"}
text2 := issueText(issue2)
if text2 != "Just title" {
t.Errorf("issueText() = %q, want %q", text2, "Just title")
}
}