diff --git a/BENCHMARKS.md b/BENCHMARKS.md index e431e218a..7387b6f27 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -41,6 +41,8 @@ Tests on graphs with different topologies (linear chains, trees, dense graphs): ### Search Operations - **BenchmarkSearchIssues_Large_NoFilter** - Search all open issues (10K dataset) - **BenchmarkSearchIssues_Large_ComplexFilter** - Search with priority/status filters (10K dataset) +- **BenchmarkPerfSearchTypedLabelFilter_5K** - Label/type search over a 5K issue/label catalog +- **BenchmarkPerfResolvePartialIDInvalidInput_5K** - Invalid partial-ID rejection without a broad fallback scan ### CRUD Operations - **BenchmarkCreateIssue_Large** - Create new issue in 10K database @@ -51,6 +53,54 @@ Tests on graphs with different topologies (linear chains, trees, dense graphs): - **BenchmarkLargeDescription** - Handling 100KB+ issue descriptions (NEW) - **BenchmarkSyncMerge** - Simulate sync cycle with create/update operations (NEW) +### Recent Perf Regression References + +These benchmarks cover the May 2026 Dolt hot-path changes so future perf PRs can run before/after checks against the same fixture shapes: + +| PR / change | Benchmark | +|-------------|-----------| +| #3966 `perf(deps): narrow recursive cycle checks` | `BenchmarkPerfAddDependencyCycleCheck_DiamondDAG` | +| #3967 `perf(search): tighten label and partial-id queries` | `BenchmarkPerfSearchTypedLabelFilter_5K`, `BenchmarkPerfResolvePartialIDInvalidInput_5K` | +| #3968 `perf(ready): page blocked checks for limited ready work` | `BenchmarkPerfReadyWorkLimited_LargeBlockedGraph` | +| #4001 `perf(ready): narrow deferred-parent child filtering` | `BenchmarkPerfReadyWorkDeferredParentExclusion_5K` | +| #4002 `perf(ready): restrict blocked dependency scans to active IDs` | `BenchmarkPerfBlockedIssues_ClosedDependencySkew` | +| #4003 `perf(get): query primary issues before wisp fallback` | `BenchmarkPerfGetIssuePrimaryFirst_PermanentWithWisps` | +| #4004 `perf(deps): scan one cycle table for same-storage edges` | No standalone executable perf diff in the landed squash; covered by the cycle-check benchmark above | + +Measured with `-benchtime=1x -benchmem -count=1` on the same host, copying this benchmark file onto each before/after ref: + +| PR / path | Benchmark | Before | After | Time gain | Alloc gain | +|-----------|-----------|--------|-------|-----------|------------| +| #3967 label/type search | `BenchmarkPerfSearchTypedLabelFilter_5K` | 134.8 ms | 51.8 ms | 61.6% | -0.1% | +| #3967 invalid partial-ID fallback | `BenchmarkPerfResolvePartialIDInvalidInput_5K` | 124.3 ms | 22.5 ms | 81.9% | 43.6% | +| #3966 dependency cycle check | `BenchmarkPerfAddDependencyCycleCheck_DiamondDAG` | 80.0 ms | 25.8 ms | 67.7% | 1.4% | +| #3968 limited ready work | `BenchmarkPerfReadyWorkLimited_LargeBlockedGraph` | 1677.4 ms | 341.7 ms | 79.6% | 85.4% | +| #4001 deferred parent exclusion | `BenchmarkPerfReadyWorkDeferredParentExclusion_5K` | 3257.3 ms | 130.8 ms | 96.0% | 83.1% | +| #4002 active blocked-dep scan | `BenchmarkPerfBlockedIssues_ClosedDependencySkew` | 44.3 ms | 36.2 ms | 18.1% | 96.0% | +| #4003 primary issue lookup | `BenchmarkPerfGetIssuePrimaryFirst_PermanentWithWisps` | 9.0 ms | 6.4 ms | 28.7% | 10.7% | + +Run the recent perf reference set with: + +```bash +go test -run=^$ -bench='BenchmarkPerf(SearchTypedLabelFilter|ResolvePartialIDInvalidInput|AddDependencyCycleCheck|ReadyWorkLimited|BlockedIssues|ReadyWorkDeferredParentExclusion|GetIssuePrimaryFirst)' -benchtime=1x -benchmem ./internal/storage/dolt +``` + +For production-shaped CLI timeout and index experiments, use: + +```bash +go run ./scripts/repro-dolt-prod-timeouts --bd ./bd --scenario all +go run ./scripts/bench-ready-indexes --dsn 'root@tcp(127.0.0.1:33307)/mc?timeout=30s&readTimeout=30s&writeTimeout=30s' +``` + +When `repro-dolt-prod-timeouts` targets an existing workspace with +`--workspace`, fixture seeding defaults to `--seed-mode=none`; pass +`--seed-mode=full` or `--seed-mode=dep-only` only when intentionally writing +and committing synthetic fixture rows into that workspace. + +`bench-ready-indexes` drops its candidate indexes again before exit by default; +pass `--keep-indexes` only when intentionally leaving the final index set +installed. + ## Performance Targets ### Typical Results (M2 Pro) diff --git a/internal/storage/dolt/dolt_benchmark_test.go b/internal/storage/dolt/dolt_benchmark_test.go index 648360387..c5e9614a8 100644 --- a/internal/storage/dolt/dolt_benchmark_test.go +++ b/internal/storage/dolt/dolt_benchmark_test.go @@ -11,14 +11,23 @@ package dolt import ( "context" + "database/sql" "fmt" "os" + "strings" "sync" "testing" + "time" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/utils" ) +func benchDatabaseName() string { + return fmt.Sprintf("beads_test_bench_%d", time.Now().UnixNano()) +} + // setupBenchStore creates a store for benchmarks func setupBenchStore(b *testing.B) (*DoltStore, func()) { b.Helper() @@ -42,7 +51,7 @@ func setupBenchStore(b *testing.B) (*DoltStore, func()) { Path: tmpDir, CommitterName: "bench", CommitterEmail: "bench@example.com", - Database: "benchdb", + Database: benchDatabaseName(), CreateIfMissing: true, } @@ -94,7 +103,7 @@ func BenchmarkBootstrapEmbedded(b *testing.B) { Path: tmpDir, CommitterName: "bench", CommitterEmail: "bench@example.com", - Database: "benchdb", + Database: benchDatabaseName(), CreateIfMissing: true, } @@ -136,13 +145,14 @@ func BenchmarkColdStart(b *testing.B) { // Get the path for reopening tmpDir := store.dbPath + dbName := store.database store.Close() cfg := &Config{ Path: tmpDir, CommitterName: "bench", CommitterEmail: "bench@example.com", - Database: "benchdb", + Database: dbName, CreateIfMissing: true, } @@ -220,13 +230,14 @@ func BenchmarkCLIWorkflow(b *testing.B) { } tmpDir := store.dbPath + dbName := store.database store.Close() cfg := &Config{ Path: tmpDir, CommitterName: "bench", CommitterEmail: "bench@example.com", - Database: "benchdb", + Database: dbName, CreateIfMissing: true, } @@ -909,6 +920,534 @@ func BenchmarkGetReadyWork(b *testing.B) { } } +// ============================================================================= +// Recent production hot-path regression benchmarks (May 2026 perf stack) +// ============================================================================= + +func createBenchIssueBatch(b *testing.B, store *DoltStore, issues []*types.Issue) { + b.Helper() + ctx := context.Background() + now := time.Now().UTC() + err := store.withRetryTx(ctx, func(tx *sql.Tx) error { + issuesByTable := map[string][]*types.Issue{ + "issues": nil, + "wisps": nil, + } + for _, issue := range issues { + if issue.Status == "" { + issue.Status = types.StatusOpen + } + if issue.Priority == 0 { + issue.Priority = 2 + } + if issue.IssueType == "" { + issue.IssueType = types.TypeTask + } + if issue.CreatedAt.IsZero() { + issue.CreatedAt = now + } + if issue.UpdatedAt.IsZero() { + issue.UpdatedAt = issue.CreatedAt + } + issue.ContentHash = issue.ComputeContentHash() + + issueTable, _ := issueops.TableRouting(issue) + issuesByTable[issueTable] = append(issuesByTable[issueTable], issue) + } + for table, tableIssues := range issuesByTable { + if err := insertBenchIssues(ctx, tx, table, tableIssues); err != nil { + return err + } + if err := insertBenchLabels(ctx, tx, table, tableIssues); err != nil { + return err + } + } + + if err := insertBenchDependencies(ctx, tx, "dependencies", issuesByTable["issues"], now); err != nil { + return err + } + if err := insertBenchDependencies(ctx, tx, "wisp_dependencies", issuesByTable["wisps"], now); err != nil { + return err + } + + return nil + }) + if err != nil { + b.Fatalf("failed to create %d benchmark issues: %v", len(issues), err) + } +} + +func insertBenchIssues(ctx context.Context, tx *sql.Tx, table string, issues []*types.Issue) error { + const batchSize = 500 + for start := 0; start < len(issues); start += batchSize { + end := start + batchSize + if end > len(issues) { + end = len(issues) + } + var placeholders []string + var args []interface{} + for _, issue := range issues[start:end] { + placeholders = append(placeholders, "(?, ?, ?, '', '', '', '', ?, ?, ?, ?, ?, ?, ?, ?, ?)") + args = append(args, + issue.ID, + issue.ContentHash, + issue.Title, + issue.Status, + issue.Priority, + issue.IssueType, + issue.CreatedAt, + issue.UpdatedAt, + issue.DeferUntil, + issue.Ephemeral, + issue.NoHistory, + issue.Pinned, + ) + } + //nolint:gosec // G201: table is selected from fixed issue table names. + query := fmt.Sprintf(` + INSERT INTO %s ( + id, content_hash, title, description, design, acceptance_criteria, notes, + status, priority, issue_type, created_at, updated_at, defer_until, + ephemeral, no_history, pinned + ) VALUES %s + ON DUPLICATE KEY UPDATE title = VALUES(title) + `, table, strings.Join(placeholders, ",")) + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("insert benchmark issues into %s: %w", table, err) + } + } + return nil +} + +func insertBenchLabels(ctx context.Context, tx *sql.Tx, issueTable string, issues []*types.Issue) error { + const batchSize = 500 + labelTable := "labels" + if issueTable == "wisps" { + labelTable = "wisp_labels" + } + var issueIDs []string + var labels []string + for _, issue := range issues { + for _, label := range issue.Labels { + issueIDs = append(issueIDs, issue.ID) + labels = append(labels, label) + } + } + for start := 0; start < len(labels); start += batchSize { + end := start + batchSize + if end > len(labels) { + end = len(labels) + } + var placeholders []string + var args []interface{} + for i := start; i < end; i++ { + placeholders = append(placeholders, "(?, ?)") + args = append(args, issueIDs[i], labels[i]) + } + //nolint:gosec // G201: labelTable is selected from fixed label table names. + query := fmt.Sprintf(` + INSERT INTO %s (issue_id, label) + VALUES %s + ON DUPLICATE KEY UPDATE label = label + `, labelTable, strings.Join(placeholders, ",")) + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("insert benchmark labels into %s: %w", labelTable, err) + } + } + return nil +} + +func insertBenchDependencies(ctx context.Context, tx *sql.Tx, depTable string, issues []*types.Issue, now time.Time) error { + type benchDependencyRow struct { + issueID string + dependsOnID string + depType types.DependencyType + createdAt time.Time + } + + const batchSize = 500 + rowsByColumn := map[string][]benchDependencyRow{} + for _, issue := range issues { + for _, dep := range issue.Dependencies { + if dep.IssueID == "" { + dep.IssueID = issue.ID + } + createdAt := dep.CreatedAt + if createdAt.IsZero() { + createdAt = now + } + column := issueops.ClassifyDepTarget(ctx, tx, dep, false).Column() + rowsByColumn[column] = append(rowsByColumn[column], benchDependencyRow{ + issueID: dep.IssueID, + dependsOnID: dep.DependsOnID, + depType: dep.Type, + createdAt: createdAt, + }) + } + } + for column, rows := range rowsByColumn { + for start := 0; start < len(rows); start += batchSize { + end := start + batchSize + if end > len(rows) { + end = len(rows) + } + var placeholders []string + var args []interface{} + for _, row := range rows[start:end] { + placeholders = append(placeholders, "(?, ?, ?, 'bench', ?, '{}')") + args = append(args, row.issueID, row.dependsOnID, row.depType, row.createdAt) + } + //nolint:gosec // G201: depTable is fixed and column comes from issueops.DepTargetKind.Column. + query := fmt.Sprintf(` + INSERT INTO %s (issue_id, %s, type, created_by, created_at, metadata) + VALUES %s + ON DUPLICATE KEY UPDATE type = type + `, depTable, column, strings.Join(placeholders, ",")) + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("insert benchmark dependencies into %s: %w", depTable, err) + } + } + } + return nil +} + +func BenchmarkPerfSearchTypedLabelFilter_5K(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const total = 5000 + issues := make([]*types.Issue, 0, total) + for i := 0; i < total; i++ { + labels := []string{fmt.Sprintf("bucket-%03d", i%500)} + issueType := types.TypeBug + if i%3 == 0 { + issueType = types.TypeFeature + } + if i%40 == 0 { + issueType = types.TypeTask + labels = append(labels, "perf-hot") + } + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-search-%05d", i), + Title: fmt.Sprintf("Search label benchmark issue %05d", i), + Status: types.StatusOpen, + Priority: (i % 4) + 1, + IssueType: issueType, + Labels: labels, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + taskType := types.TypeTask + filter := types.IssueFilter{IssueType: &taskType, Labels: []string{"perf-hot"}, Limit: 200} + b.ResetTimer() + for i := 0; i < b.N; i++ { + results, err := store.SearchIssues(ctx, "", filter) + if err != nil { + b.Fatalf("SearchIssues label filter: %v", err) + } + if len(results) == 0 { + b.Fatal("SearchIssues label filter returned no issues") + } + } +} + +func BenchmarkPerfResolvePartialIDInvalidInput_5K(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const total = 5000 + issues := make([]*types.Issue, 0, total) + for i := 0; i < total; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-partial-%05d", i), + Title: fmt.Sprintf("Partial ID benchmark issue %05d", i), + Status: types.StatusOpen, + Priority: (i % 4) + 1, + IssueType: types.TypeTask, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := utils.ResolvePartialID(ctx, store, "not/a/valid/id"); err == nil { + b.Fatal("ResolvePartialID invalid input unexpectedly succeeded") + } + } +} + +func BenchmarkPerfAddDependencyCycleCheck_DiamondDAG(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const layers = 10 + issues := make([]*types.Issue, 0, 2*layers+1) + issues = append(issues, &types.Issue{ + ID: "bench-perf-cycle-source", + Title: "Cycle source", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }) + for layer := 0; layer < layers; layer++ { + for _, suffix := range []string{"a", "b"} { + issue := &types.Issue{ + ID: fmt.Sprintf("bench-perf-cycle-l%02d-%s", layer, suffix), + Title: fmt.Sprintf("Cycle diamond layer %02d %s", layer, suffix), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if layer < layers-1 { + issue.Dependencies = []*types.Dependency{ + {DependsOnID: fmt.Sprintf("bench-perf-cycle-l%02d-a", layer+1), Type: types.DepBlocks}, + {DependsOnID: fmt.Sprintf("bench-perf-cycle-l%02d-b", layer+1), Type: types.DepBlocks}, + } + } + issues = append(issues, issue) + } + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + dep := &types.Dependency{ + IssueID: "bench-perf-cycle-source", + DependsOnID: "bench-perf-cycle-l00-a", + Type: types.DepBlocks, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := store.AddDependency(ctx, dep, "bench"); err != nil { + b.Fatalf("AddDependency cycle check: %v", err) + } + b.StopTimer() + if err := store.RemoveDependency(ctx, dep.IssueID, dep.DependsOnID, "bench"); err != nil { + b.Fatalf("RemoveDependency cycle check fixture reset: %v", err) + } + if i < b.N-1 { + b.StartTimer() + } + } +} + +func BenchmarkPerfReadyWorkLimited_LargeBlockedGraph(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const ( + readyCount = 100 + blockedPairCount = 2500 + estimatedRowCount = readyCount + 2*blockedPairCount + ) + issues := make([]*types.Issue, 0, estimatedRowCount) + for i := 0; i < readyCount; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-ready-clear-%04d", i), + Title: fmt.Sprintf("Ready issue %04d", i), + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + }) + } + for i := 0; i < blockedPairCount; i++ { + blockerID := fmt.Sprintf("bench-perf-ready-blocker-%05d", i) + issues = append(issues, &types.Issue{ + ID: blockerID, + Title: fmt.Sprintf("Ready blocker %05d", i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }) + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-ready-blocked-%05d", i), + Title: fmt.Sprintf("Blocked ready issue %05d", i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: blockerID, Type: types.DepBlocks}, + }, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + filter := types.WorkFilter{Limit: 50, SortPolicy: types.SortPolicyPriority} + b.ResetTimer() + for i := 0; i < b.N; i++ { + results, err := store.GetReadyWork(ctx, filter) + if err != nil { + b.Fatalf("GetReadyWork limited blocked graph: %v", err) + } + if len(results) != filter.Limit { + b.Fatalf("GetReadyWork limited blocked graph returned %d issues, want %d", len(results), filter.Limit) + } + } +} + +func BenchmarkPerfBlockedIssues_ClosedDependencySkew(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const ( + closedPairCount = 2500 + activePairCount = 10 + ) + issues := make([]*types.Issue, 0, 2*(closedPairCount+activePairCount)) + for i := 0; i < closedPairCount; i++ { + blockerID := fmt.Sprintf("bench-perf-closed-blocker-%05d", i) + issues = append(issues, &types.Issue{ + ID: blockerID, + Title: fmt.Sprintf("Closed blocker %05d", i), + Status: types.StatusClosed, + Priority: 2, + IssueType: types.TypeTask, + }) + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-closed-blocked-%05d", i), + Title: fmt.Sprintf("Closed blocked issue %05d", i), + Status: types.StatusClosed, + Priority: 2, + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: blockerID, Type: types.DepBlocks}, + }, + }) + } + for i := 0; i < activePairCount; i++ { + blockerID := fmt.Sprintf("bench-perf-active-blocker-%02d", i) + issues = append(issues, &types.Issue{ + ID: blockerID, + Title: fmt.Sprintf("Active blocker %02d", i), + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + }) + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-active-blocked-%02d", i), + Title: fmt.Sprintf("Active blocked issue %02d", i), + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: blockerID, Type: types.DepBlocks}, + }, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + b.ResetTimer() + for i := 0; i < b.N; i++ { + results, err := store.GetBlockedIssues(ctx, types.WorkFilter{}) + if err != nil { + b.Fatalf("GetBlockedIssues closed skew: %v", err) + } + if len(results) != activePairCount { + b.Fatalf("GetBlockedIssues closed skew returned %d issues, want %d", len(results), activePairCount) + } + } +} + +func BenchmarkPerfReadyWorkDeferredParentExclusion_5K(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + future := time.Now().UTC().Add(24 * time.Hour) + const ( + readyCount = 100 + deferredParentCount = 5000 + deferredChildCount = 50 + ) + issues := make([]*types.Issue, 0, readyCount+deferredParentCount+deferredChildCount) + for i := 0; i < readyCount; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-deferred-ready-%03d", i), + Title: fmt.Sprintf("Ready non-deferred issue %03d", i), + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + }) + } + for i := 0; i < deferredParentCount; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-deferred-parent-%05d", i), + Title: fmt.Sprintf("Future deferred parent %05d", i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + DeferUntil: &future, + }) + } + for i := 0; i < deferredChildCount; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-deferred-child-%03d", i), + Title: fmt.Sprintf("Child of deferred parent %03d", i), + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: fmt.Sprintf("bench-perf-deferred-parent-%05d", i), Type: types.DepParentChild}, + }, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + filter := types.WorkFilter{Limit: 50, SortPolicy: types.SortPolicyPriority} + b.ResetTimer() + for i := 0; i < b.N; i++ { + results, err := store.GetReadyWork(ctx, filter) + if err != nil { + b.Fatalf("GetReadyWork deferred parent exclusion: %v", err) + } + if len(results) != filter.Limit { + b.Fatalf("GetReadyWork deferred parent exclusion returned %d issues, want %d", len(results), filter.Limit) + } + } +} + +func BenchmarkPerfGetIssuePrimaryFirst_PermanentWithWisps(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const wispCount = 5000 + issues := make([]*types.Issue, 0, wispCount+1) + issues = append(issues, &types.Issue{ + ID: "bench-perf-get-primary", + Title: "Permanent issue fetched from primary table", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + }) + for i := 0; i < wispCount; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("bench-perf-get-wisp-%04d", i), + Title: fmt.Sprintf("Wisp noise issue %04d", i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: true, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + b.ResetTimer() + for i := 0; i < b.N; i++ { + issue, err := store.GetIssue(ctx, "bench-perf-get-primary") + if err != nil { + b.Fatalf("GetIssue permanent with wisps: %v", err) + } + if issue == nil || issue.ID != "bench-perf-get-primary" { + b.Fatalf("GetIssue permanent with wisps returned %v", issue) + } + } +} + // ============================================================================= // Label Benchmarks // ============================================================================= diff --git a/scripts/bench-ready-indexes/main.go b/scripts/bench-ready-indexes/main.go new file mode 100644 index 000000000..69200ee82 --- /dev/null +++ b/scripts/bench-ready-indexes/main.go @@ -0,0 +1,303 @@ +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + "sync" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +type candidateIndex struct { + Name string + Table string + SQL string + Drop string +} + +type queryCase struct { + Name string + Query string + Args []any +} + +type result struct { + Name string + Count int + Errors int + Min time.Duration + P50 time.Duration + P95 time.Duration + Max time.Duration + Elapsed time.Duration +} + +var indexes = []candidateIndex{ + { + Name: "idx_dependencies_type_issue_target", + Table: "dependencies", + SQL: "CREATE INDEX idx_dependencies_type_issue_target ON dependencies (type, issue_id, depends_on_issue_id)", + Drop: "DROP INDEX idx_dependencies_type_issue_target ON dependencies", + }, + { + Name: "idx_issues_ready_assignee", + Table: "issues", + SQL: "CREATE INDEX idx_issues_ready_assignee ON issues (assignee, status, priority, created_at, id)", + Drop: "DROP INDEX idx_issues_ready_assignee ON issues", + }, + { + Name: "idx_issues_ready_status", + Table: "issues", + SQL: "CREATE INDEX idx_issues_ready_status ON issues (status, priority, created_at, id)", + Drop: "DROP INDEX idx_issues_ready_status ON issues", + }, + { + Name: "idx_issues_defer_until", + Table: "issues", + SQL: "CREATE INDEX idx_issues_defer_until ON issues (defer_until)", + Drop: "DROP INDEX idx_issues_defer_until ON issues", + }, +} + +var sqlOpen = sql.Open + +const dependencyTargetExpr = "COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external)" + +func main() { + if err := run(context.Background()); err != nil { + log.Print(err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + return runWithArgs(ctx, os.Args[1:]) +} + +func runWithArgs(ctx context.Context, args []string) error { + var dsn string + var concurrency int + var iterations int + var keepIndexes bool + fs := flag.NewFlagSet("bench-ready-indexes", flag.ContinueOnError) + fs.StringVar(&dsn, "dsn", "root@tcp(127.0.0.1:33307)/mc?timeout=30s&readTimeout=30s&writeTimeout=30s", "MySQL DSN") + fs.IntVar(&concurrency, "concurrency", 64, "concurrent query workers") + fs.IntVar(&iterations, "iterations", 2, "query executions per worker per case") + fs.BoolVar(&keepIndexes, "keep-indexes", false, "leave candidate indexes installed after the final benchmark state") + if err := fs.Parse(args); err != nil { + return err + } + + db, err := sqlOpen("mysql", dsn) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(concurrency + 8) + db.SetMaxIdleConns(concurrency + 8) + if err := db.PingContext(ctx); err != nil { + return err + } + defer func() { + if err := cleanupCandidateIndexes(context.Background(), db, keepIndexes); err != nil { + log.Printf("cleanup candidate indexes: %v", err) + } + }() + + queries, err := buildQueries(ctx, db) + if err != nil { + return err + } + + states := [][]candidateIndex{nil} + for _, idx := range indexes { + states = append(states, []candidateIndex{idx}) + } + states = append(states, indexes) + + for _, state := range states { + if err := dropAll(ctx, db); err != nil { + return err + } + stateName := "baseline" + if len(state) > 0 { + names := make([]string, 0, len(state)) + for _, idx := range state { + start := time.Now() + if _, err := db.ExecContext(ctx, idx.SQL); err != nil { + return fmt.Errorf("create %s: %w", idx.Name, err) + } + fmt.Printf("created %s in %s\n", idx.Name, time.Since(start).Round(time.Millisecond)) + names = append(names, idx.Name) + } + stateName = strings.Join(names, "+") + } + + fmt.Printf("\n## %s\n", stateName) + for _, qc := range queries { + r := benchQuery(ctx, db, qc, concurrency, iterations) + fmt.Printf("%-32s count=%4d errors=%2d min=%7s p50=%7s p95=%7s max=%7s wall=%7s\n", + r.Name, r.Count, r.Errors, r.Min.Round(time.Millisecond), r.P50.Round(time.Millisecond), + r.P95.Round(time.Millisecond), r.Max.Round(time.Millisecond), r.Elapsed.Round(time.Millisecond)) + } + } + return nil +} + +func cleanupCandidateIndexes(ctx context.Context, db *sql.DB, keepIndexes bool) error { + if keepIndexes { + return nil + } + return dropAll(ctx, db) +} + +func dropAll(ctx context.Context, db *sql.DB) error { + for _, idx := range indexes { + if _, err := db.ExecContext(ctx, idx.Drop); err != nil && !strings.Contains(err.Error(), "not found") && !strings.Contains(err.Error(), "does not exist") && !strings.Contains(err.Error(), "can't drop") { + return fmt.Errorf("drop %s: %w", idx.Name, err) + } + } + return nil +} + +func buildQueries(ctx context.Context, db *sql.DB) ([]queryCase, error) { + ids, err := candidateIDs(ctx, db) + if err != nil { + return nil, err + } + placeholders := strings.TrimRight(strings.Repeat("?,", len(ids)), ",") + args := make([]any, 0, len(ids)) + for _, id := range ids { + args = append(args, id) + } + return []queryCase{ + { + Name: "ready_assignee_page", + Query: `SELECT id FROM issues +WHERE status IN ('open','in_progress') +AND (pinned = 0 OR pinned IS NULL) +AND (ephemeral = 0 OR ephemeral IS NULL) +AND id IN (SELECT id FROM issues WHERE issue_type NOT IN ('merge-request','gate','molecule','message','agent','role','rig')) +AND assignee = 'gascity--control-dispatcher' +AND (defer_until IS NULL OR defer_until <= UTC_TIMESTAMP()) +ORDER BY priority ASC, created_at DESC, id ASC +LIMIT 100`, + }, + { + Name: "ready_routed_page", + Query: `SELECT id FROM issues +WHERE status IN ('open','in_progress') +AND (pinned = 0 OR pinned IS NULL) +AND (ephemeral = 0 OR ephemeral IS NULL) +AND id IN (SELECT id FROM issues WHERE issue_type NOT IN ('merge-request','gate','molecule','message','agent','role','rig')) +AND (assignee IS NULL OR assignee = '') +AND (defer_until IS NULL OR defer_until <= UTC_TIMESTAMP()) +AND JSON_UNQUOTE(JSON_EXTRACT(metadata, '$."gc.routed_to"')) = 'gascity/control-dispatcher' +ORDER BY priority ASC, created_at DESC, id ASC +LIMIT 100`, + }, + { + Name: "deferred_parents", + Query: `SELECT id FROM issues +WHERE defer_until IS NOT NULL AND defer_until > UTC_TIMESTAMP()`, + }, + { + Name: "candidate_blocking_deps", + Query: fmt.Sprintf(`SELECT issue_id, %s AS target_id, type, metadata FROM dependencies WHERE issue_id IN (%s) AND type IN ('blocks','waits-for','conditional-blocks')`, dependencyTargetExpr, placeholders), + Args: args, + }, + }, nil +} + +func candidateIDs(ctx context.Context, db *sql.DB) ([]string, error) { + rows, err := db.QueryContext(ctx, `SELECT id FROM issues WHERE status IN ('open','in_progress') ORDER BY priority ASC, created_at DESC, id ASC LIMIT 100`) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, fmt.Errorf("no candidate IDs") + } + return ids, nil +} + +func benchQuery(ctx context.Context, db *sql.DB, qc queryCase, concurrency, iterations int) result { + start := time.Now() + var mu sync.Mutex + var latencies []time.Duration + errors := 0 + var wg sync.WaitGroup + for worker := 0; worker < concurrency; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + qStart := time.Now() + rows, err := db.QueryContext(ctx, qc.Query, qc.Args...) + if err == nil { + for rows.Next() { + } + if rows.Err() != nil { + err = rows.Err() + } + _ = rows.Close() + } + latency := time.Since(qStart) + mu.Lock() + if err != nil { + errors++ + } else { + latencies = append(latencies, latency) + } + mu.Unlock() + } + }() + } + wg.Wait() + sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] }) + return result{ + Name: qc.Name, + Count: len(latencies), + Errors: errors, + Min: percentile(latencies, 0), + P50: percentile(latencies, 50), + P95: percentile(latencies, 95), + Max: percentile(latencies, 100), + Elapsed: time.Since(start), + } +} + +func percentile(values []time.Duration, pct int) time.Duration { + if len(values) == 0 { + return 0 + } + if pct <= 0 { + return values[0] + } + if pct >= 100 { + return values[len(values)-1] + } + idx := (len(values)*pct + 99) / 100 + if idx < 1 { + idx = 1 + } + return values[idx-1] +} diff --git a/scripts/bench-ready-indexes/main_test.go b/scripts/bench-ready-indexes/main_test.go new file mode 100644 index 000000000..ded51347a --- /dev/null +++ b/scripts/bench-ready-indexes/main_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestCleanupCandidateIndexesDropsByDefault(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + for _, idx := range indexes { + mock.ExpectExec(regexp.QuoteMeta(idx.Drop)). + WillReturnResult(sqlmock.NewResult(0, 0)) + } + + if err := cleanupCandidateIndexes(context.Background(), db, false); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestCleanupCandidateIndexesHonorsKeepIndexes(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if err := cleanupCandidateIndexes(context.Background(), db, true); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestRunCleansUpCandidateIndexesAfterCreateFailure(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + + oldSQLOpen := sqlOpen + sqlOpen = func(driverName, dataSourceName string) (*sql.DB, error) { + if driverName != "mysql" { + t.Fatalf("driverName = %q, want mysql", driverName) + } + if dataSourceName != "sqlmock-dsn" { + t.Fatalf("dataSourceName = %q, want sqlmock-dsn", dataSourceName) + } + return db, nil + } + t.Cleanup(func() { + sqlOpen = oldSQLOpen + }) + + mock.ExpectPing() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT id FROM issues WHERE status IN ('open','in_progress') ORDER BY priority ASC, created_at DESC, id ASC LIMIT 100`)). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("bd-a")) + + expectDropAll(mock) // baseline + for _, idx := range indexes { + expectDropAll(mock) + mock.ExpectExec(regexp.QuoteMeta(idx.SQL)). + WillReturnResult(sqlmock.NewResult(0, 0)) + } + expectDropAll(mock) // all-indexes state starts from a clean slate + mock.ExpectExec(regexp.QuoteMeta(indexes[0].SQL)). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta(indexes[1].SQL)). + WillReturnError(errors.New("create failed")) + expectDropAll(mock) // deferred cleanup still runs after the create error + mock.ExpectClose() + + err = runWithArgs(context.Background(), []string{ + "--dsn", "sqlmock-dsn", + "--concurrency", "0", + "--iterations", "1", + }) + if err == nil { + t.Fatal("expected create-index error") + } + if !strings.Contains(err.Error(), "create "+indexes[1].Name) { + t.Fatalf("error = %v, want create failure for %s", err, indexes[1].Name) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestBuildQueriesUsesTypedDependencyTargetProjection(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT id FROM issues WHERE status IN ('open','in_progress') ORDER BY priority ASC, created_at DESC, id ASC LIMIT 100`)). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("bd-a")) + + queries, err := buildQueries(context.Background(), db) + if err != nil { + t.Fatal(err) + } + var found bool + for _, qc := range queries { + if qc.Name != "candidate_blocking_deps" { + continue + } + found = true + if !strings.Contains(qc.Query, dependencyTargetExpr) { + t.Fatalf("candidate query = %q, want typed target projection", qc.Query) + } + if strings.Contains(qc.Query, "depends_on"+"_id") { + t.Fatalf("candidate query still references generated column: %q", qc.Query) + } + } + if !found { + t.Fatal("candidate_blocking_deps query not found") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func expectDropAll(mock sqlmock.Sqlmock) { + for _, idx := range indexes { + mock.ExpectExec(regexp.QuoteMeta(idx.Drop)). + WillReturnResult(sqlmock.NewResult(0, 0)) + } +} diff --git a/scripts/repro-dolt-prod-timeouts/main.go b/scripts/repro-dolt-prod-timeouts/main.go new file mode 100644 index 000000000..4ed0be043 --- /dev/null +++ b/scripts/repro-dolt-prod-timeouts/main.go @@ -0,0 +1,1149 @@ +// repro-dolt-prod-timeouts runs production-shaped bd CLI timeout scenarios. +// +// It initializes a real server-mode beads workspace, bulk-loads a graph that +// mirrors maintainer-city's skew (large mostly-closed issue table, large +// dependency table, small active frontier), then forks actual bd commands. +// +// Usage: +// +// go run ./scripts/repro-dolt-prod-timeouts --bd ./bd --scenario all +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/storage/doltutil" +) + +type config struct { + BDPath string + Workspace string + SeedMode string + Scenario string + IssueCount int + DepCount int + Concurrency int + Ops int + Timeout time.Duration + ChainDepth int + KeepWorkdir bool + ManagedServer bool +} + +type workspace struct { + Dir string + BeadsDir string + Port int + Database string +} + +type opResult struct { + Kind string `json:"kind"` + Argv []string `json:"argv"` + Latency time.Duration `json:"latency"` + TimedOut bool `json:"timed_out"` + Err string `json:"err,omitempty"` + StderrTail string `json:"stderr_tail,omitempty"` +} + +type job struct { + Kind string + Argv []string + Env []string + Sh string +} + +var subprocessEnvDenylist = []string{ + "BEADS_DIR", + "BEADS_DOLT_SERVER_PORT", + "BEADS_DOLT_PORT", + "BEADS_DOLT_SERVER_HOST", + "BEADS_DOLT_SERVER_SOCKET", +} + +const dependencyTargetExpr = "COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external)" + +func main() { + if err := run(context.Background()); err != nil { + log.Print(err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + return runWithArgs(ctx, os.Args[1:]) +} + +func runWithArgs(ctx context.Context, args []string) error { + cfg, err := parseFlags(args) + if err != nil { + return err + } + bdPath, err := resolveExecutablePath(cfg.BDPath) + if err != nil { + return err + } + cfg.BDPath = bdPath + + ws, err := openOrCreateWorkspace(ctx, cfg) + if err != nil { + return err + } + defer func() { + if cfg.Workspace == "" { + stopWorkspace(context.Background(), cfg, ws) + } + if cfg.KeepWorkdir { + fmt.Printf("kept workdir: %s\n", ws.Dir) + return + } + if cfg.Workspace == "" { + _ = os.RemoveAll(ws.Dir) + } + }() + + fmt.Printf("workspace=%s port=%d database=%s\n", ws.Dir, ws.Port, ws.Database) + if err := loadProductionShape(ctx, ws, cfg); err != nil { + return err + } + + scenarios, err := scenarioNames(cfg.Scenario) + if err != nil { + return err + } + for _, scenario := range scenarios { + report(scenario, runScenario(ctx, cfg, ws, scenario)) + } + return nil +} + +func parseFlags(args []string) (config, error) { + var cfg config + fs := flag.NewFlagSet("repro-dolt-prod-timeouts", flag.ContinueOnError) + fs.StringVar(&cfg.BDPath, "bd", "bd", "bd binary to execute") + fs.StringVar(&cfg.Workspace, "workspace", "", "existing workspace to test instead of creating a synthetic one") + fs.StringVar(&cfg.SeedMode, "seed-mode", "full", "fixture seed mode: full, dep-only, none") + fs.StringVar(&cfg.Scenario, "scenario", "all", "scenario: ready, dep, control, mixed, outage, cycle-current, cycle-deps-only, cycle-wisps-only, cycle-bfs, all") + fs.IntVar(&cfg.IssueCount, "issues", 100000, "issue rows to seed") + fs.IntVar(&cfg.DepCount, "deps", 85000, "dependency rows to seed") + fs.IntVar(&cfg.Concurrency, "concurrency", 20, "concurrent bd processes") + fs.IntVar(&cfg.Ops, "ops", 80, "total operations per scenario") + fs.DurationVar(&cfg.Timeout, "timeout", 30*time.Second, "per-command timeout") + fs.IntVar(&cfg.ChainDepth, "chain-depth", 0, "existing blocking chain depth behind each dep-add target") + fs.BoolVar(&cfg.KeepWorkdir, "keep-workdir", false, "keep temp workspace") + if err := fs.Parse(args); err != nil { + return config{}, err + } + if cfg.Workspace != "" && !flagPassed(fs, "seed-mode") { + cfg.SeedMode = "none" + } + return cfg, nil +} + +func flagPassed(fs *flag.FlagSet, name string) bool { + passed := false + fs.Visit(func(f *flag.Flag) { + if f.Name == name { + passed = true + } + }) + return passed +} + +func resolveExecutablePath(path string) (string, error) { + bdPath, err := exec.LookPath(path) + if err != nil { + return "", fmt.Errorf("find bd binary %q: %w", path, err) + } + absPath, err := filepath.Abs(bdPath) + if err != nil { + return "", fmt.Errorf("resolve bd binary %q: %w", bdPath, err) + } + return absPath, nil +} + +var allScenarioNames = []string{ + "ready", + "dep", + "control", + "mixed", + "outage", + "cycle-current", + "cycle-deps-only", + "cycle-wisps-only", + "cycle-bfs", +} + +func scenarioNames(scenario string) ([]string, error) { + if scenario == "all" { + return append([]string(nil), allScenarioNames...), nil + } + for _, name := range allScenarioNames { + if scenario == name { + return []string{scenario}, nil + } + } + return nil, fmt.Errorf("unknown scenario %q", scenario) +} + +func runScenario(ctx context.Context, cfg config, ws *workspace, scenario string) []opResult { + switch scenario { + case "ready": + return runReadyScenario(ctx, cfg, ws) + case "dep": + return runDepScenario(ctx, cfg, ws) + case "control": + return runControlQueryScenario(ctx, cfg, ws) + case "mixed": + return runMixedCityLoadScenario(ctx, cfg, ws) + case "outage": + return runOutageScenario(ctx, cfg, ws) + case "cycle-current": + return runCycleCheckScenario(ctx, cfg, ws, cycleCheckCurrentSQL) + case "cycle-deps-only": + return runCycleCheckScenario(ctx, cfg, ws, cycleCheckDependenciesOnlySQL) + case "cycle-wisps-only": + return runCycleCheckScenario(ctx, cfg, ws, cycleCheckWispsOnlySQL) + case "cycle-bfs": + return runCycleCheckScenario(ctx, cfg, ws, cycleCheckBatchedBFS) + default: + return []opResult{{Kind: scenario, Err: fmt.Sprintf("unknown scenario %q", scenario)}} + } +} + +func openOrCreateWorkspace(ctx context.Context, cfg config) (*workspace, error) { + if cfg.Workspace != "" { + return openWorkspace(ctx, cfg, cfg.Workspace) + } + return createWorkspace(ctx, cfg) +} + +func openWorkspace(ctx context.Context, cfg config, dir string) (*workspace, error) { + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + beadsDir := filepath.Join(absDir, ".beads") + if _, err := os.Stat(beadsDir); err != nil { + return nil, fmt.Errorf("open .beads in %s: %w", absDir, err) + } + port, err := readInt(filepath.Join(beadsDir, "dolt-server.port")) + if err != nil { + return nil, fmt.Errorf("read dolt-server.port: %w", err) + } + if !isPortOpen(port) { + if err := startWorkspaceDolt(ctx, cfg, absDir); err != nil { + return nil, err + } + port, err = readInt(filepath.Join(beadsDir, "dolt-server.port")) + if err != nil { + return nil, fmt.Errorf("read dolt-server.port: %w", err) + } + } + database, err := readDoltDatabase(filepath.Join(beadsDir, "metadata.json")) + if err != nil { + return nil, err + } + return &workspace{Dir: absDir, BeadsDir: beadsDir, Port: port, Database: database}, nil +} + +func isPortOpen(port int) bool { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), time.Second) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +func createWorkspace(ctx context.Context, cfg config) (*workspace, error) { + dir, err := os.MkdirTemp("", "bd-prod-timeout-*") + if err != nil { + return nil, err + } + + initTimeout := cfg.Timeout * 4 + if initTimeout < 2*time.Minute { + initTimeout = 2 * time.Minute + } + initCtx, cancel := context.WithTimeout(ctx, initTimeout) + defer cancel() + + fmt.Printf("initializing server workspace timeout=%s\n", initTimeout) + cmd := exec.CommandContext(initCtx, cfg.BDPath, + "init", + "--server", + "--prefix=perf", + "--non-interactive", + "--quiet", + "--skip-hooks", + "--skip-agents", + ) + cmd.Dir = dir + cmd.Env = subprocessEnv("BD_NON_INTERACTIVE=1") + if out, err := cmd.CombinedOutput(); err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("bd init after %s: %w\n%s", initTimeout, err, string(out)) + } + + beadsDir := filepath.Join(dir, ".beads") + port, err := readInt(filepath.Join(beadsDir, "dolt-server.port")) + if err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("read dolt-server.port: %w", err) + } + database, err := readDoltDatabase(filepath.Join(beadsDir, "metadata.json")) + if err != nil { + _ = os.RemoveAll(dir) + return nil, err + } + return &workspace{Dir: dir, BeadsDir: beadsDir, Port: port, Database: database}, nil +} + +func startWorkspaceDolt(ctx context.Context, cfg config, dir string) error { + startTimeout := cfg.Timeout * 2 + if startTimeout < time.Minute { + startTimeout = time.Minute + } + startCtx, cancel := context.WithTimeout(ctx, startTimeout) + defer cancel() + + cmd := exec.CommandContext(startCtx, cfg.BDPath, "dolt", "start") + cmd.Dir = dir + cmd.Env = subprocessEnv("BD_NON_INTERACTIVE=1") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("bd dolt start after %s: %w\n%s", startTimeout, err, string(out)) + } + return nil +} + +func readDoltDatabase(path string) (string, error) { + //nolint:gosec // G304: benchmark harness reads metadata from the selected workspace. + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read metadata.json: %w", err) + } + var meta struct { + DoltDatabase string `json:"dolt_database"` + } + if err := json.Unmarshal(data, &meta); err != nil { + return "", fmt.Errorf("parse metadata.json: %w", err) + } + if meta.DoltDatabase == "" { + return "", fmt.Errorf("metadata.json missing dolt_database") + } + return meta.DoltDatabase, nil +} + +func stopWorkspace(ctx context.Context, cfg config, ws *workspace) { + cmd := exec.CommandContext(ctx, cfg.BDPath, "dolt", "stop") + cmd.Dir = ws.Dir + cmd.Env = subprocessEnv("BD_NON_INTERACTIVE=1") + _ = cmd.Run() +} + +func loadProductionShape(ctx context.Context, ws *workspace, cfg config) error { + if cfg.SeedMode == "none" { + fmt.Printf("seed skipped seed_mode=%s\n", cfg.SeedMode) + return nil + } + + dsn := doltutil.ServerDSN{ + Host: "127.0.0.1", + Port: ws.Port, + User: "root", + Database: ws.Database, + Timeout: cfg.Timeout, + }.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + + start := time.Now() + if err := seedProductionShape(ctx, db, cfg); err != nil { + return err + } + fmt.Printf("seeded mode=%s issues=%d deps=%d in %s\n", cfg.SeedMode, cfg.IssueCount, cfg.DepCount, time.Since(start).Round(time.Millisecond)) + return nil +} + +func seedProductionShape(ctx context.Context, db *sql.DB, cfg config) error { + switch cfg.SeedMode { + case "full": + if err := insertIssues(ctx, db, cfg.IssueCount, cfg.Ops, cfg.ChainDepth); err != nil { + return err + } + if err := insertDependencies(ctx, db, cfg.DepCount, cfg.IssueCount); err != nil { + return err + } + case "dep-only": + if err := insertDepIssues(ctx, db, cfg.Ops, cfg.ChainDepth); err != nil { + return err + } + default: + return fmt.Errorf("unknown seed mode %q", cfg.SeedMode) + } + if cfg.ChainDepth > 0 { + if err := insertDepAddChains(ctx, db, cfg.Ops, cfg.ChainDepth); err != nil { + return err + } + } + if _, err := db.ExecContext(ctx, "CALL DOLT_ADD('-A')"); err != nil { + return fmt.Errorf("DOLT_ADD fixture: %w", err) + } + if _, err := db.ExecContext(ctx, "CALL DOLT_COMMIT('-m', 'seed production timeout fixture')"); err != nil { + return fmt.Errorf("DOLT_COMMIT fixture: %w", err) + } + return nil +} + +func insertIssues(ctx context.Context, db *sql.DB, count, depOps, chainDepth int) error { + const batchSize = 500 + depIssueCount := depOps * 2 + if chainDepth > 0 { + depIssueCount = depOps*(chainDepth+3) + chainDepth + 2 + } + total := count + depIssueCount + for start := 0; start < total; start += batchSize { + end := start + batchSize + if end > total { + end = total + } + var q strings.Builder + q.WriteString(`INSERT INTO issues + (id, title, description, design, acceptance_criteria, notes, + status, priority, issue_type, assignee, metadata) + VALUES `) + args := make([]any, 0, (end-start)*11) + for i := start; i < end; i++ { + if i > start { + q.WriteByte(',') + } + q.WriteString("(?,?,?,?,?,?,?,?,?,?,?)") + id := fmt.Sprintf("perf-%06d", i) + status := "closed" + assignee := "" + metadata := "{}" + if i < 350 { + status = "open" + } + if i < 40 { + assignee = "gascity--control-dispatcher" + } + if i >= 40 && i < 80 { + metadata = `{"gc.routed_to":"gascity/control-dispatcher"}` + } + if i >= count { + status = "open" + id = depIssueID(i - count) + } + args = append(args, id, fmt.Sprintf("prod timeout issue %d", i), "fixture", "", "", "", status, (i%4)+1, "task", assignee, metadata) + } + if _, err := db.ExecContext(ctx, q.String(), args...); err != nil { + return fmt.Errorf("insert issues %d-%d: %w", start, end, err) + } + } + return nil +} + +func insertDepIssues(ctx context.Context, db *sql.DB, depOps, chainDepth int) error { + const batchSize = 500 + total := depFixtureIssueCount(depOps, chainDepth) + for start := 0; start < total; start += batchSize { + end := start + batchSize + if end > total { + end = total + } + var q strings.Builder + q.WriteString(`INSERT INTO issues + (id, title, description, design, acceptance_criteria, notes, + status, priority, issue_type, assignee, metadata) + VALUES `) + args := make([]any, 0, (end-start)*11) + for i := start; i < end; i++ { + if i > start { + q.WriteByte(',') + } + q.WriteString("(?,?,?,?,?,?,?,?,?,?,?)") + args = append(args, depIssueID(i), fmt.Sprintf("prod copy dep issue %d", i), "fixture", "", "", "", "open", (i%4)+1, "task", "", "{}") + } + if _, err := db.ExecContext(ctx, q.String(), args...); err != nil { + return fmt.Errorf("insert dep issues %d-%d: %w", start, end, err) + } + } + return nil +} + +func depFixtureIssueCount(depOps, chainDepth int) int { + if chainDepth > 0 { + return depOps*(chainDepth+3) + chainDepth + 2 + } + return depOps * 2 +} + +func insertDependencies(ctx context.Context, db *sql.DB, count, issueCount int) error { + const batchSize = 500 + if count <= 0 { + return nil + } + maxPairs, err := maxDependencyPairs(issueCount) + if err != nil { + return err + } + if count > maxPairs { + return fmt.Errorf("dependency count %d exceeds unique dependency pairs for %d issues", count, issueCount) + } + for start := 0; start < count; start += batchSize { + end := start + batchSize + if end > count { + end = count + } + var q strings.Builder + q.WriteString(`INSERT INTO dependencies + (issue_id, depends_on_issue_id, type, created_by, metadata) + VALUES `) + args := make([]any, 0, (end-start)*5) + for i := start; i < end; i++ { + if i > start { + q.WriteByte(',') + } + q.WriteString("(?,?,?,?,?)") + issueID, dependsOnID := dependencyEndpoints(i, issueCount, 1000, 300) + depType := "parent-child" + if i < 20 || (i >= 40 && i < 60) { + issueID, dependsOnID = dependencyEndpoints(i, issueCount, 0, 200) + depType = "blocks" + } else if i < 5000 { + depType = "blocks" + } + args = append(args, issueID, dependsOnID, depType, "bench", "{}") + } + if _, err := db.ExecContext(ctx, q.String(), args...); err != nil { + return fmt.Errorf("insert dependencies %d-%d: %w", start, end, err) + } + } + return nil +} + +func maxDependencyPairs(issueCount int) (int, error) { + if issueCount <= 0 { + return 0, fmt.Errorf("issue count must be positive, got %d", issueCount) + } + if issueCount == 1 { + return 1, nil + } + return issueCount * (issueCount - 1), nil +} + +func dependencyEndpoints(i, issueCount, sourceOffset, targetOffset int) (string, string) { + if issueCount == 1 { + return perfIssueID(0), perfIssueID(0) + } + source := (i + sourceOffset) % issueCount + round := i / issueCount + offset := 1 + ((targetOffset - 1 + round) % (issueCount - 1)) + target := (source + offset) % issueCount + return perfIssueID(source), perfIssueID(target) +} + +func perfIssueID(i int) string { + return fmt.Sprintf("perf-%06d", i) +} + +func insertDepAddChains(ctx context.Context, db *sql.DB, ops, depth int) error { + const batchSize = 500 + if depth <= 0 || ops <= 0 { + return nil + } + + total := ops * depth + for start := 0; start < total; start += batchSize { + end := start + batchSize + if end > total { + end = total + } + var q strings.Builder + q.WriteString(`INSERT INTO dependencies + (issue_id, depends_on_issue_id, type, created_by, metadata) + VALUES `) + args := make([]any, 0, (end-start)*5) + for i := start; i < end; i++ { + if i > start { + q.WriteByte(',') + } + q.WriteString("(?,?,?,?,?)") + op := i / depth + step := i % depth + base := depBase(op, depth) + issueID := depIssueID(base + 1 + step) + dependsOnID := depIssueID(base + 2 + step) + args = append(args, issueID, dependsOnID, "blocks", "bench", "{}") + } + if _, err := db.ExecContext(ctx, q.String(), args...); err != nil { + return fmt.Errorf("insert dep-add chains %d-%d: %w", start, end, err) + } + } + return nil +} + +func runReadyScenario(ctx context.Context, cfg config, ws *workspace) []opResult { + jobs := make([]job, 0, cfg.Ops) + for i := 0; i < cfg.Ops; i++ { + if i%2 == 0 { + jobs = append(jobs, job{Kind: "ready", Argv: []string{"ready", "--assignee=gascity--control-dispatcher", "--json", "--limit=20"}}) + } else { + jobs = append(jobs, job{Kind: "ready", Argv: []string{"ready", "--metadata-field", "gc.routed_to=gascity/control-dispatcher", "--unassigned", "--json", "--limit=20"}}) + } + } + return runJobs(ctx, cfg, ws, jobs) +} + +func runDepScenario(ctx context.Context, cfg config, ws *workspace) []opResult { + jobs := make([]job, 0, cfg.Ops) + for i := 0; i < cfg.Ops; i++ { + jobs = append(jobs, depAddJob(i, cfg.ChainDepth)) + } + return runJobs(ctx, cfg, ws, jobs) +} + +func runControlQueryScenario(ctx context.Context, cfg config, ws *workspace) []opResult { + return runJobs(ctx, cfg, ws, controlQueryJobs(cfg.Ops)) +} + +func runOutageScenario(ctx context.Context, cfg config, ws *workspace) []opResult { + jobs := make([]job, 0, cfg.Ops*2) + jobs = append(jobs, controlQueryJobs(cfg.Ops)...) + for i := 0; i < cfg.Ops; i++ { + jobs = append(jobs, depAddJob(i, cfg.ChainDepth)) + } + return runJobs(ctx, cfg, ws, jobs) +} + +type cycleCheckFunc func(context.Context, *sql.DB, string, string, int) error + +func runCycleCheckScenario(ctx context.Context, cfg config, ws *workspace, check cycleCheckFunc) []opResult { + dsn := doltutil.ServerDSN{ + Host: "127.0.0.1", + Port: ws.Port, + User: "root", + Database: ws.Database, + Timeout: cfg.Timeout, + }.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + return []opResult{{Kind: "cycle-open", Err: err.Error()}} + } + defer db.Close() + db.SetMaxOpenConns(cfg.Concurrency) + db.SetMaxIdleConns(cfg.Concurrency) + + start := time.Now() + jobCh := make(chan int) + resCh := make(chan opResult, cfg.Ops) + var wg sync.WaitGroup + for w := 0; w < cfg.Concurrency; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range jobCh { + opCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) + base := depBase(i, cfg.ChainDepth) + issueID := depIssueID(base) + dependsOnID := depIssueID(base + 1) + opStart := time.Now() + err := check(opCtx, db, issueID, dependsOnID, cfg.ChainDepth) + latency := time.Since(opStart) + res := opResult{Kind: "cycle", Argv: []string{"cycle-check", issueID, dependsOnID}, Latency: latency} + if opCtx.Err() == context.DeadlineExceeded { + res.TimedOut = true + } + if err != nil { + res.Err = err.Error() + res.StderrTail = tail(err.Error(), 300) + } + cancel() + resCh <- res + } + }() + } + for i := 0; i < cfg.Ops; i++ { + jobCh <- i + } + close(jobCh) + wg.Wait() + close(resCh) + + results := make([]opResult, 0, cfg.Ops) + for res := range resCh { + results = append(results, res) + } + fmt.Printf("scenario wall=%s\n", time.Since(start).Round(time.Millisecond)) + return results +} + +func cycleCheckCurrentSQL(ctx context.Context, db *sql.DB, issueID, dependsOnID string, _ int) error { + var reachable int + query := fmt.Sprintf(` + WITH RECURSIVE reachable AS ( + SELECT ? AS node, 0 AS depth + UNION ALL + SELECT d.target_id, r.depth + 1 + FROM reachable r + JOIN ( + SELECT issue_id, %s AS target_id FROM dependencies WHERE type IN ('blocks', 'conditional-blocks') + UNION ALL + SELECT issue_id, %s AS target_id FROM wisp_dependencies WHERE type IN ('blocks', 'conditional-blocks') + ) d ON d.issue_id = r.node + WHERE r.depth < 100 + ) + SELECT COUNT(*) FROM reachable WHERE node = ? + `, dependencyTargetExpr, dependencyTargetExpr) + err := db.QueryRowContext(ctx, query, dependsOnID, issueID).Scan(&reachable) + if err != nil { + return err + } + if reachable > 0 { + return fmt.Errorf("cycle detected") + } + return nil +} + +func cycleCheckDependenciesOnlySQL(ctx context.Context, db *sql.DB, issueID, dependsOnID string, _ int) error { + return cycleCheckOneTableSQL(ctx, db, "dependencies", issueID, dependsOnID) +} + +func cycleCheckWispsOnlySQL(ctx context.Context, db *sql.DB, issueID, dependsOnID string, _ int) error { + return cycleCheckOneTableSQL(ctx, db, "wisp_dependencies", issueID, dependsOnID) +} + +func cycleCheckOneTableSQL(ctx context.Context, db *sql.DB, table, issueID, dependsOnID string) error { + var reachable int + //nolint:gosec // G201: table is selected by fixed scenario wrappers. + query := fmt.Sprintf(` + WITH RECURSIVE reachable AS ( + SELECT ? AS node, 0 AS depth + UNION ALL + SELECT %s, r.depth + 1 + FROM reachable r + JOIN %s d ON d.issue_id = r.node + WHERE d.type IN ('blocks', 'conditional-blocks') AND r.depth < 100 + ) + SELECT COUNT(*) FROM reachable WHERE node = ? + `, dependencyTargetExpr, table) + if err := db.QueryRowContext(ctx, query, dependsOnID, issueID).Scan(&reachable); err != nil { + return err + } + if reachable > 0 { + return fmt.Errorf("cycle detected") + } + return nil +} + +func cycleCheckBatchedBFS(ctx context.Context, db *sql.DB, issueID, dependsOnID string, maxDepth int) error { + if maxDepth <= 0 || maxDepth > 100 { + maxDepth = 100 + } + seen := map[string]struct{}{dependsOnID: {}} + frontier := []string{dependsOnID} + for depth := 0; depth < maxDepth && len(frontier) > 0; depth++ { + next, err := fetchBlockingTargets(ctx, db, frontier) + if err != nil { + return err + } + frontier = frontier[:0] + for _, id := range next { + if id == issueID { + return fmt.Errorf("cycle detected") + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + frontier = append(frontier, id) + } + } + return nil +} + +func fetchBlockingTargets(ctx context.Context, db *sql.DB, issueIDs []string) ([]string, error) { + if len(issueIDs) == 0 { + return nil, nil + } + args := make([]any, 0, len(issueIDs)*2) + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(issueIDs)), ",") + //nolint:gosec // G201: placeholders are generated from ? markers only. + query := fmt.Sprintf(` + SELECT %s FROM dependencies + WHERE issue_id IN (%s) AND type IN ('blocks', 'conditional-blocks') + UNION ALL + SELECT %s FROM wisp_dependencies + WHERE issue_id IN (%s) AND type IN ('blocks', 'conditional-blocks') + `, dependencyTargetExpr, placeholders, dependencyTargetExpr, placeholders) + for _, id := range issueIDs { + args = append(args, id) + } + for _, id := range issueIDs { + args = append(args, id) + } + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + +func runMixedCityLoadScenario(ctx context.Context, cfg config, ws *workspace) []opResult { + start := time.Now() + depCount := cfg.Ops + backgroundCount := cfg.Ops * cfg.Concurrency + + results := make([]opResult, 0, depCount+backgroundCount) + resultCh := make(chan opResult, depCount+backgroundCount) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < depCount; i++ { + job := depAddJob(i, cfg.ChainDepth) + job.Kind = "dispatcher-dep" + resultCh <- runJob(ctx, cfg, ws, job) + } + }() + + backgroundJobs := mixedBackgroundJobs(backgroundCount) + jobCh := make(chan job) + for w := 0; w < cfg.Concurrency; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for job := range jobCh { + resultCh <- runJob(ctx, cfg, ws, job) + } + }() + } + for _, job := range backgroundJobs { + jobCh <- job + } + close(jobCh) + + wg.Wait() + close(resultCh) + for res := range resultCh { + results = append(results, res) + } + fmt.Printf("scenario wall=%s\n", time.Since(start).Round(time.Millisecond)) + return results +} + +func mixedBackgroundJobs(count int) []job { + jobs := make([]job, 0, count) + for i := 0; i < count; i++ { + switch i % 6 { + case 0: + jobs = append(jobs, job{Kind: "session-ready", Argv: []string{"ready", "--include-ephemeral", "--assignee=" + sessionAssignee(i), "--json", "--limit=1"}}) + case 1: + jobs = append(jobs, job{Kind: "control-ready", Argv: []string{"--readonly", "--sandbox", "ready", "--include-ephemeral", "--assignee=gascity--control-dispatcher", "--json", "--limit=20"}}) + case 2: + jobs = append(jobs, job{Kind: "route-ready", Argv: []string{"--readonly", "--sandbox", "ready", "--include-ephemeral", "--metadata-field", "gc.routed_to=gascity/control-dispatcher", "--unassigned", "--json", "--limit=20"}}) + case 3: + jobs = append(jobs, job{Kind: "show", Argv: []string{"show", fmt.Sprintf("perf-%06d", i%350), "--json"}}) + case 4: + jobs = append(jobs, job{Kind: "list", Argv: []string{"list", "--json", "--status", "in_progress", "--assignee=" + sessionAssignee(i), "--limit=1"}}) + case 5: + jobs = append(jobs, job{Kind: "claim", Argv: []string{"update", fmt.Sprintf("perf-%06d", i%40), "--claim", "--json"}}) + } + } + return jobs +} + +func sessionAssignee(i int) string { + return fmt.Sprintf("mc-%07d", i%64) +} + +func depAddJob(i, chainDepth int) job { + base := depBase(i, chainDepth) + return job{ + Kind: "dep", + Argv: []string{"dep", "add", depIssueID(base), depIssueID(base + 1), "--type", "blocks", "--json"}, + } +} + +func depBase(i, chainDepth int) int { + if chainDepth <= 0 { + return i * 2 + } + return i * (chainDepth + 3) +} + +func controlQueryJobs(count int) []job { + targets := []struct { + target string + session string + legacy string + }{ + {target: "control-dispatcher", session: "control-dispatcher", legacy: "workflow-control"}, + {target: "gascity/control-dispatcher", session: "gascity--control-dispatcher", legacy: "gascity/workflow-control"}, + {target: "gasworks-gui/control-dispatcher", session: "gasworks-gui--control-dispatcher", legacy: "gasworks-gui/workflow-control"}, + {target: "gtest-rig/control-dispatcher", session: "gtest-rig--control-dispatcher", legacy: "gtest-rig/workflow-control"}, + } + + jobs := make([]job, 0, count) + script := controlQueryScript() + for i := 0; i < count; i++ { + t := targets[i%len(targets)] + jobs = append(jobs, job{ + Kind: "control", + Sh: script, + Env: []string{ + "BD_EXPORT_AUTO=false", + "GC_CONTROL_TARGET=" + t.target, + "GC_CONTROL_SESSION_NAME=" + t.session, + "GC_CONTROL_LEGACY_TARGET=" + t.legacy, + "GC_SESSION_NAME=" + t.session, + "GC_ALIAS=" + t.target, + "GC_SESSION_ID=" + t.session, + }, + }) + } + return jobs +} + +func controlQueryScript() string { + return `BD_EXPORT_AUTO=false +tmp=$(mktemp) +err=$(mktemp) +trap "rm -f \"$tmp\" \"$err\"" EXIT +emit_ready() { + r=$("$@" 2>"$err") + status=$? + if [ "$status" -ne 0 ]; then + printf "ready probe failed (%s):\n" "$*" >&2 + cat "$err" >&2 + return "$status" + fi + [ -n "$r" ] && [ "$r" != "[]" ] && printf "%s\n" "$r" >> "$tmp" +} +for id in "$GC_CONTROL_SESSION_NAME" "$GC_SESSION_NAME" "$GC_ALIAS" "$GC_CONTROL_TARGET" "$GC_SESSION_ID"; do + [ -z "$id" ] && continue + legacy="" + case "$id" in *control-dispatcher) legacy="${id%control-dispatcher}workflow-control";; esac + for cand in "$id" "$legacy"; do + [ -z "$cand" ] && continue + emit_ready "$BD_BIN" --readonly --sandbox ready --assignee="$cand" --json --limit=20 || exit $? + done +done +emit_ready "$BD_BIN" --readonly --sandbox ready --metadata-field "gc.routed_to=$GC_CONTROL_TARGET" --unassigned --json --limit=20 || exit $? +emit_ready "$BD_BIN" --readonly --sandbox ready --metadata-field "gc.routed_to=$GC_CONTROL_LEGACY_TARGET" --unassigned --json --limit=20 || exit $? +[ -s "$tmp" ] && jq -s 'reduce add[] as $item ([]; if any(.[]; .id == $item.id) then . else . + [$item] end)' "$tmp" || printf "[]" +` +} + +func depIssueID(i int) string { + return fmt.Sprintf("perf-dep-%06d", i) +} + +func runJobs(ctx context.Context, cfg config, ws *workspace, jobs []job) []opResult { + start := time.Now() + jobCh := make(chan job) + resCh := make(chan opResult, len(jobs)) + var wg sync.WaitGroup + for w := 0; w < cfg.Concurrency; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for job := range jobCh { + resCh <- runJob(ctx, cfg, ws, job) + } + }() + } + for _, job := range jobs { + jobCh <- job + } + close(jobCh) + wg.Wait() + close(resCh) + + results := make([]opResult, 0, len(jobs)) + for res := range resCh { + results = append(results, res) + } + fmt.Printf("scenario wall=%s\n", time.Since(start).Round(time.Millisecond)) + return results +} + +func runJob(ctx context.Context, cfg config, ws *workspace, j job) opResult { + if j.Sh != "" { + return runShell(ctx, cfg, ws, j) + } + return runBD(ctx, cfg, ws, j) +} + +func runBD(ctx context.Context, cfg config, ws *workspace, j job) opResult { + opCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) + defer cancel() + cmd := exec.CommandContext(opCtx, cfg.BDPath, j.Argv...) + cmd.Dir = ws.Dir + cmd.Env = subprocessEnv(append([]string{"BD_NON_INTERACTIVE=1"}, j.Env...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + start := time.Now() + err := cmd.Run() + latency := time.Since(start) + res := opResult{Kind: j.Kind, Argv: j.Argv, Latency: latency} + if len(res.Argv) == 0 { + res.Argv = []string{"sh", "-c", compactShell(j.Sh)} + } + if opCtx.Err() == context.DeadlineExceeded { + res.TimedOut = true + } + if err != nil { + res.Err = err.Error() + } + res.StderrTail = tail(stderr.String(), 300) + return res +} + +func runShell(ctx context.Context, cfg config, ws *workspace, j job) opResult { + opCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) + defer cancel() + cmd := exec.CommandContext(opCtx, "sh", "-c", j.Sh) + cmd.Dir = ws.Dir + cmd.Env = subprocessEnv(append([]string{"BD_NON_INTERACTIVE=1", "BD_BIN=" + cfg.BDPath}, j.Env...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + start := time.Now() + err := cmd.Run() + latency := time.Since(start) + res := opResult{Kind: j.Kind, Argv: []string{"sh", "-c", compactShell(j.Sh)}, Latency: latency} + if opCtx.Err() == context.DeadlineExceeded { + res.TimedOut = true + } + if err != nil { + res.Err = err.Error() + } + res.StderrTail = tail(stderr.String(), 300) + return res +} + +func compactShell(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +func report(name string, results []opResult) { + sort.Slice(results, func(i, j int) bool { return results[i].Latency < results[j].Latency }) + var failures, timeouts int + var driverReadTimeouts int + for _, r := range results { + if r.Err != "" { + failures++ + } + if r.TimedOut { + timeouts++ + } + if isDriverReadTimeout(r) { + driverReadTimeouts++ + } + } + fmt.Printf("\n[%s] ops=%d failures=%d harness_timeouts=%d driver_read_timeouts=%d p50=%s p95=%s max=%s\n", + name, len(results), failures, timeouts, driverReadTimeouts, + percentile(results, 50).Round(time.Millisecond), + percentile(results, 95).Round(time.Millisecond), + percentile(results, 100).Round(time.Millisecond), + ) + for i := len(results) - 1; i >= 0 && i >= len(results)-5; i-- { + r := results[i] + fmt.Printf(" slow kind=%s latency=%s timeout=%t err=%q stderr=%q argv=%s\n", + r.Kind, r.Latency.Round(time.Millisecond), r.TimedOut, r.Err, r.StderrTail, strings.Join(r.Argv, " ")) + } +} + +func isDriverReadTimeout(r opResult) bool { + return strings.Contains(r.StderrTail, "packets.go:58 read tcp") && + strings.Contains(r.StderrTail, "i/o timeout") +} + +func percentile(results []opResult, p int) time.Duration { + if len(results) == 0 { + return 0 + } + if p >= 100 { + return results[len(results)-1].Latency + } + idx := (len(results)*p + 99) / 100 + if idx <= 0 { + idx = 1 + } + return results[idx-1].Latency +} + +func readInt(path string) (int, error) { + //nolint:gosec // G304: benchmark harness reads control files from the selected workspace. + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + return strconv.Atoi(strings.TrimSpace(string(data))) +} + +func subprocessEnv(extra ...string) []string { + env := cleanEnv(os.Environ(), subprocessEnvDenylist...) + return append(env, extra...) +} + +func cleanEnv(env []string, keys ...string) []string { + drop := make(map[string]struct{}, len(keys)) + for _, key := range keys { + drop[key] = struct{}{} + } + out := env[:0] + for _, e := range env { + key, _, ok := strings.Cut(e, "=") + if ok { + if _, skip := drop[key]; skip { + continue + } + } + out = append(out, e) + } + return out +} + +func tail(s string, max int) string { + s = strings.TrimSpace(s) + if len(s) <= max { + return s + } + return s[len(s)-max:] +} diff --git a/scripts/repro-dolt-prod-timeouts/main_test.go b/scripts/repro-dolt-prod-timeouts/main_test.go new file mode 100644 index 000000000..2cee0597e --- /dev/null +++ b/scripts/repro-dolt-prod-timeouts/main_test.go @@ -0,0 +1,437 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "slices" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/testutil" +) + +func TestIsDriverReadTimeout(t *testing.T) { + result := opResult{ + StderrTail: "[mysql] 2026/05/13 18:13:48 packets.go:58 read tcp 127.0.0.1:39308->127.0.0.1:21791: i/o timeout", + } + if !isDriverReadTimeout(result) { + t.Fatal("expected MySQL driver read timeout to be classified") + } +} + +func TestIsDriverReadTimeoutIgnoresHarnessTimeout(t *testing.T) { + result := opResult{ + TimedOut: true, + StderrTail: "signal: killed", + } + if isDriverReadTimeout(result) { + t.Fatal("harness timeout should not be classified as driver read timeout") + } +} + +func TestMixedBackgroundJobsIncludesSessionLoadShapes(t *testing.T) { + jobs := mixedBackgroundJobs(12) + seen := map[string]bool{} + for _, job := range jobs { + seen[job.Kind] = true + } + for _, want := range []string{"session-ready", "control-ready", "route-ready", "show", "list", "claim"} { + if !seen[want] { + t.Fatalf("mixed background jobs missing %q; seen=%v", want, seen) + } + } +} + +func TestDepFixtureIssueCountIncludesChainTargets(t *testing.T) { + if got := depFixtureIssueCount(10, 0); got != 20 { + t.Fatalf("without chains got %d, want 20", got) + } + if got := depFixtureIssueCount(10, 100); got != 1132 { + t.Fatalf("with chains got %d, want 1132", got) + } +} + +func TestResolveExecutablePathReturnsAbsolutePath(t *testing.T) { + tmp := t.TempDir() + bd := filepath.Join(tmp, "bd") + if err := os.WriteFile(bd, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(tmp) + + got, err := resolveExecutablePath("./bd") + if err != nil { + t.Fatal(err) + } + if got != bd { + t.Fatalf("got %q, want %q", got, bd) + } +} + +func TestParseFlagsDefaultsSyntheticWorkspaceToFullSeed(t *testing.T) { + cfg, err := parseFlags(nil) + if err != nil { + t.Fatal(err) + } + if cfg.SeedMode != "full" { + t.Fatalf("SeedMode = %q, want full", cfg.SeedMode) + } +} + +func TestParseFlagsDefaultsExistingWorkspaceToNoSeed(t *testing.T) { + cfg, err := parseFlags([]string{"--workspace", "/tmp/existing-beads-workspace"}) + if err != nil { + t.Fatal(err) + } + if cfg.SeedMode != "none" { + t.Fatalf("SeedMode = %q, want none", cfg.SeedMode) + } +} + +func TestParseFlagsPreservesExplicitExistingWorkspaceSeedMode(t *testing.T) { + cfg, err := parseFlags([]string{"--workspace", "/tmp/existing-beads-workspace", "--seed-mode", "full"}) + if err != nil { + t.Fatal(err) + } + if cfg.SeedMode != "full" { + t.Fatalf("SeedMode = %q, want full", cfg.SeedMode) + } +} + +func TestBenchmarkSubprocessesStripDoltEnvOverrides(t *testing.T) { + for _, key := range subprocessEnvDenylist { + t.Setenv(key, "caller-"+strings.ToLower(key)) + } + + tmp := t.TempDir() + bd := filepath.Join(tmp, "bd") + if err := os.WriteFile(bd, []byte("#!/bin/sh\n"+noDoltOverrideEnvScript()), 0o755); err != nil { + t.Fatal(err) + } + + cfg := config{BDPath: bd, Timeout: time.Second} + ws := &workspace{Dir: tmp} + if got := runBD(context.Background(), cfg, ws, job{Kind: "env", Argv: []string{"status"}}); got.Err != "" { + t.Fatalf("runBD inherited denied env: err=%q stderr=%q", got.Err, got.StderrTail) + } + if got := runShell(context.Background(), cfg, ws, job{Kind: "env", Sh: noDoltOverrideEnvScript()}); got.Err != "" { + t.Fatalf("runShell inherited denied env: err=%q stderr=%q", got.Err, got.StderrTail) + } +} + +func TestControlQueryScriptPreservesReadyProbeFailure(t *testing.T) { + tmp := t.TempDir() + bd := filepath.Join(tmp, "bd") + if err := os.WriteFile(bd, []byte("#!/bin/sh\necho ready failed >&2\nexit 17\n"), 0o755); err != nil { + t.Fatal(err) + } + + cfg := config{BDPath: bd, Timeout: time.Second} + ws := &workspace{Dir: tmp} + result := runShell(context.Background(), cfg, ws, controlQueryJobs(1)[0]) + if result.Err == "" { + t.Fatalf("expected control query shell to fail when bd ready fails; stderr=%q", result.StderrTail) + } + if !strings.Contains(result.StderrTail, "ready failed") { + t.Fatalf("stderr tail = %q, want ready probe stderr", result.StderrTail) + } +} + +func noDoltOverrideEnvScript() string { + var b strings.Builder + b.WriteString("for key in") + for _, key := range subprocessEnvDenylist { + b.WriteByte(' ') + b.WriteString(key) + } + b.WriteString(`; do + eval "value=\${$key:-}" + if [ -n "$value" ]; then + echo "$key inherited: $value" >&2 + exit 42 + fi +done +exit 0 +`) + return b.String() +} + +func TestScenarioNamesAllIncludesEveryAdvertisedScenario(t *testing.T) { + got, err := scenarioNames("all") + if err != nil { + t.Fatal(err) + } + want := []string{ + "ready", + "dep", + "control", + "mixed", + "outage", + "cycle-current", + "cycle-deps-only", + "cycle-wisps-only", + "cycle-bfs", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("scenarioNames(all) got %v, want %v", got, want) + } +} + +func TestInsertDependenciesWritesTypedIssueTargetColumn(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectExec(`INSERT INTO dependencies\s+\(issue_id, depends_on_issue_id, type, created_by, metadata\)\s+VALUES`). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := insertDependencies(context.Background(), db, 1, 100000); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestInsertDependenciesUsesConfiguredIssueRange(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectExec(`INSERT INTO dependencies\s+\(issue_id, depends_on_issue_id, type, created_by, metadata\)\s+VALUES`). + WithArgs( + "perf-000000", "perf-000004", "blocks", "bench", "{}", + "perf-000001", "perf-000000", "blocks", "bench", "{}", + "perf-000002", "perf-000001", "blocks", "bench", "{}", + "perf-000003", "perf-000002", "blocks", "bench", "{}", + "perf-000004", "perf-000003", "blocks", "bench", "{}", + ). + WillReturnResult(sqlmock.NewResult(0, 5)) + + if err := insertDependencies(context.Background(), db, 5, 5); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestInsertDependenciesRejectsImpossibleUniquePairCount(t *testing.T) { + db, _, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + err = insertDependencies(context.Background(), db, 3, 2) + if err == nil { + t.Fatal("expected impossible pair count error") + } + if !strings.Contains(err.Error(), "exceeds unique dependency pairs") { + t.Fatalf("error = %v, want unique dependency pair failure", err) + } +} + +func TestInsertDepAddChainsWritesTypedIssueTargetColumn(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectExec(`INSERT INTO dependencies\s+\(issue_id, depends_on_issue_id, type, created_by, metadata\)\s+VALUES`). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := insertDepAddChains(context.Background(), db, 1, 1); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestCycleCheckCurrentSQLUsesTypedDependencyTargetProjection(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectQuery(regexp.QuoteMeta(dependencyTargetExpr)). + WithArgs("perf-000002", "perf-000001"). + WillReturnRows(sqlmock.NewRows([]string{"COUNT(*)"}).AddRow(0)) + + if err := cycleCheckCurrentSQL(context.Background(), db, "perf-000001", "perf-000002", 0); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestFetchBlockingTargetsUsesTypedDependencyTargetProjection(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectQuery(regexp.QuoteMeta(dependencyTargetExpr)). + WithArgs("perf-000001", "perf-000001"). + WillReturnRows(sqlmock.NewRows([]string{"target_id"}).AddRow("perf-000002")) + + got, err := fetchBlockingTargets(context.Background(), db, []string{"perf-000001"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []string{"perf-000002"}) { + t.Fatalf("targets = %v, want [perf-000002]", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestSeedProductionShapeFullCoversTypedDependencyInserts(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + mock.ExpectExec(`INSERT INTO issues\s+\(id, title, description, design, acceptance_criteria, notes,\s+status, priority, issue_type, assignee, metadata\)\s+VALUES`). + WillReturnResult(sqlmock.NewResult(0, 8)) + mock.ExpectExec(`INSERT INTO dependencies\s+\(issue_id, depends_on_issue_id, type, created_by, metadata\)\s+VALUES`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO dependencies\s+\(issue_id, depends_on_issue_id, type, created_by, metadata\)\s+VALUES`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_ADD('-A')")). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'seed production timeout fixture')")). + WillReturnResult(sqlmock.NewResult(0, 0)) + + cfg := config{SeedMode: "full", IssueCount: 1, DepCount: 1, Ops: 1, ChainDepth: 1} + if err := seedProductionShape(context.Background(), db, cfg); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestSeedProductionShapeFullSmallIssueCountRealSchema(t *testing.T) { + if _, err := exec.LookPath("dolt"); err != nil { + t.Skip("dolt not installed, skipping real-schema smoke") + } + skipOldDoltForCurrentSchema(t) + + ctx := context.Background() + baseDir := t.TempDir() + dbName := "testdb" + dbDir := filepath.Join(baseDir, dbName) + if err := os.MkdirAll(dbDir, 0o755); err != nil { + t.Fatal(err) + } + runTestCmd(t, dbDir, "dolt", "init", "--name", "test", "--email", "test@example.com") + + port, err := testutil.FindFreePort() + if err != nil { + t.Fatal(err) + } + serverCmd := exec.Command("dolt", "sql-server", + "-H", "127.0.0.1", + "-P", fmt.Sprintf("%d", port), + ) + serverCmd.Dir = baseDir + if err := serverCmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = serverCmd.Process.Kill() + _ = serverCmd.Wait() + }) + if !testutil.WaitForServer(port, 15*time.Second) { + t.Fatal("dolt sql-server did not become ready") + } + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: port, User: "root", Database: dbName, Timeout: 10 * time.Second}.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + defer db.Close() + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("migrate schema: %v", err) + } + + cfg := config{SeedMode: "full", IssueCount: 50, DepCount: 50, Ops: 5, ChainDepth: 2} + if err := seedProductionShape(ctx, db, cfg); err != nil { + t.Fatal(err) + } + + var depRows int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM dependencies").Scan(&depRows); err != nil { + t.Fatal(err) + } + wantDeps := cfg.DepCount + cfg.Ops*cfg.ChainDepth + if depRows != wantDeps { + t.Fatalf("dependency rows = %d, want %d", depRows, wantDeps) + } + + sourceID, targetID := dependencyEndpoints(0, cfg.IssueCount, 0, 200) + if err := cycleCheckCurrentSQL(ctx, db, perfIssueID(1), sourceID, 0); err != nil { + t.Fatalf("cycleCheckCurrentSQL: %v", err) + } + targets, err := fetchBlockingTargets(ctx, db, []string{sourceID}) + if err != nil { + t.Fatalf("fetchBlockingTargets: %v", err) + } + if !slices.Contains(targets, targetID) { + t.Fatalf("fetchBlockingTargets(%q) = %v, want %q", sourceID, targets, targetID) + } +} + +func skipOldDoltForCurrentSchema(t *testing.T) { + t.Helper() + output, err := exec.Command("dolt", "version").CombinedOutput() + if err != nil { + t.Skipf("dolt version unavailable, skipping real-schema smoke: %v", err) + } + if regexp.MustCompile(`\bdolt version 1\.`).Match(output) { + t.Skipf("dolt 1.x cannot initialize the current migration set: %s", strings.TrimSpace(string(output))) + } +} + +func runTestCmd(t *testing.T, dir string, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s %v failed in %s: %v\nOutput: %s", name, args, dir, err, output) + } +} + +func runTestDoltSQL(t *testing.T, dir, query string) { + t.Helper() + cmd := exec.Command("dolt", "sql", "-q", query) + cmd.Dir = dir + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt sql failed in %s: %v\nQuery: %.200s...\nOutput: %s", dir, err, query, output) + } +}