feat: auto-migration shim: SQLite to Dolt via sqlite3 CLI (bd--3y8)
Add a non-CGO migration path that uses the system sqlite3 CLI to export SQLite data as JSON, then imports into Dolt. This enables automatic SQLite→Dolt migration for users on non-CGO builds, which is the v1.0.0 upgrade path before SQLite driver removal. Changes: - migrate_shim.go: sqlite3 CLI extraction (extractViaSQLiteCLI, queryJSON) - migrate_import.go: shared types/functions moved out of CGO-only file (migrationData, importToDolt, findSQLiteDB, nullable helpers) - migrate_auto_nocgo.go: now calls the shim instead of being a no-op - migrate_dolt.go: removed functions moved to migrate_import.go - migrate_shim_test.go: tests including CGO/shim parity verification Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Executed-By: beads/polecats/quartz Rig: beads Role: polecats
This commit is contained in:
parent
f473b756ab
commit
13a8daa369
5 changed files with 1160 additions and 241 deletions
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
package main
|
||||
|
||||
// autoMigrateSQLiteToDolt is a no-op in non-CGO builds.
|
||||
// SQLite reading requires CGO; users on non-CGO builds must migrate manually.
|
||||
func autoMigrateSQLiteToDolt() {}
|
||||
// autoMigrateSQLiteToDolt uses the sqlite3 CLI shim for non-CGO builds.
|
||||
// This enables automatic SQLite→Dolt migration without requiring the
|
||||
// ncruces/go-sqlite3 CGO driver.
|
||||
func autoMigrateSQLiteToDolt() {
|
||||
shimMigrateSQLiteToDolt()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,17 +23,6 @@ import (
|
|||
_ "github.com/ncruces/go-sqlite3/embed"
|
||||
)
|
||||
|
||||
// migrationData holds all data extracted from the source database
|
||||
type migrationData struct {
|
||||
issues []*types.Issue
|
||||
labelsMap map[string][]string
|
||||
depsMap map[string][]*types.Dependency
|
||||
eventsMap map[string][]*types.Event
|
||||
config map[string]string
|
||||
prefix string
|
||||
issueCount int
|
||||
}
|
||||
|
||||
// handleToDoltMigration migrates from SQLite to Dolt backend.
|
||||
// 1. Finds SQLite .db files in .beads/
|
||||
// 2. Creates Dolt database in `.beads/dolt/`
|
||||
|
|
@ -175,30 +164,6 @@ func handleToDoltMigration(dryRun bool, autoYes bool) {
|
|||
printFinalStatus("dolt", imported, skipped, backupPath, doltPath, sqlitePath, true)
|
||||
}
|
||||
|
||||
// findSQLiteDB looks for a SQLite .db file in the beads directory.
|
||||
// Returns the path to the first .db file found, or empty string if none.
|
||||
func findSQLiteDB(beadsDir string) string {
|
||||
// Check common names first
|
||||
for _, name := range []string{"beads.db", "issues.db"} {
|
||||
p := filepath.Join(beadsDir, name)
|
||||
if info, err := os.Stat(p); err == nil && !info.IsDir() {
|
||||
return p
|
||||
}
|
||||
}
|
||||
// Scan for any .db file
|
||||
entries, err := os.ReadDir(beadsDir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".db") &&
|
||||
!strings.Contains(entry.Name(), "backup") {
|
||||
return filepath.Join(beadsDir, entry.Name())
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hooksNeedDoltUpdate checks if installed git hooks lack the Dolt backend skip logic.
|
||||
func hooksNeedDoltUpdate(beadsDir string) bool {
|
||||
repoRoot := filepath.Dir(beadsDir)
|
||||
|
|
@ -232,20 +197,9 @@ func handleToSQLiteMigration(_ bool, _ bool) {
|
|||
"Dolt is now the only storage backend")
|
||||
}
|
||||
|
||||
// parseNullTime parses a time string into *time.Time. Returns nil for empty strings.
|
||||
func parseNullTime(s string) *time.Time {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.999999999Z07:00", "2006-01-02 15:04:05"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return &t
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractFromSQLite extracts all data from a SQLite database using raw SQL.
|
||||
// This is the CGO path — it reads SQLite directly via the ncruces/go-sqlite3 driver.
|
||||
// For non-CGO builds, see migrate_shim.go which uses the sqlite3 CLI instead.
|
||||
func extractFromSQLite(ctx context.Context, dbPath string) (*migrationData, error) {
|
||||
db, err := sql.Open("sqlite3", "file:"+dbPath+"?mode=ro")
|
||||
if err != nil {
|
||||
|
|
@ -422,147 +376,7 @@ func extractFromSQLite(ctx context.Context, dbPath string) (*migrationData, erro
|
|||
}, nil
|
||||
}
|
||||
|
||||
// importToDolt imports all data to Dolt, returning (imported, skipped, error)
|
||||
func importToDolt(ctx context.Context, store *dolt.DoltStore, data *migrationData) (int, int, error) {
|
||||
// Set all config values first
|
||||
for key, value := range data.config {
|
||||
if err := store.SetConfig(ctx, key, value); err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to set config %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := store.UnderlyingDB().BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
imported := 0
|
||||
skipped := 0
|
||||
seenIDs := make(map[string]bool)
|
||||
total := len(data.issues)
|
||||
|
||||
for i, issue := range data.issues {
|
||||
if !jsonOutput && total > 100 && (i+1)%100 == 0 {
|
||||
fmt.Printf(" Importing issues: %d/%d\r", i+1, total)
|
||||
}
|
||||
|
||||
if seenIDs[issue.ID] {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
seenIDs[issue.ID] = true
|
||||
|
||||
if issue.ContentHash == "" {
|
||||
issue.ContentHash = issue.ComputeContentHash()
|
||||
}
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO issues (
|
||||
id, content_hash, title, description, design, acceptance_criteria, notes,
|
||||
status, priority, issue_type, assignee, estimated_minutes,
|
||||
created_at, created_by, owner, updated_at, closed_at, external_ref,
|
||||
compaction_level, compacted_at, compacted_at_commit, original_size,
|
||||
sender, ephemeral, pinned, is_template, crystallizes,
|
||||
mol_type, work_type, quality_score, source_system, source_repo, close_reason,
|
||||
event_kind, actor, target, payload,
|
||||
await_type, await_id, timeout_ns, waiters,
|
||||
hook_bead, role_bead, agent_state, last_activity, role_type, rig,
|
||||
due_at, defer_until
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?
|
||||
)
|
||||
`,
|
||||
issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes,
|
||||
issue.Status, issue.Priority, issue.IssueType, nullableString(issue.Assignee), nullableIntPtr(issue.EstimatedMinutes),
|
||||
issue.CreatedAt, issue.CreatedBy, issue.Owner, issue.UpdatedAt, issue.ClosedAt, nullableStringPtr(issue.ExternalRef),
|
||||
issue.CompactionLevel, issue.CompactedAt, nullableStringPtr(issue.CompactedAtCommit), nullableInt(issue.OriginalSize),
|
||||
issue.Sender, issue.Ephemeral, issue.Pinned, issue.IsTemplate, issue.Crystallizes,
|
||||
issue.MolType, issue.WorkType, nullableFloat32Ptr(issue.QualityScore), issue.SourceSystem, issue.SourceRepo, issue.CloseReason,
|
||||
issue.EventKind, issue.Actor, issue.Target, issue.Payload,
|
||||
issue.AwaitType, issue.AwaitID, issue.Timeout.Nanoseconds(), formatJSONArray(issue.Waiters),
|
||||
issue.HookBead, issue.RoleBead, issue.AgentState, issue.LastActivity, issue.RoleType, issue.Rig,
|
||||
issue.DueAt, issue.DeferUntil,
|
||||
)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Duplicate entry") ||
|
||||
strings.Contains(err.Error(), "UNIQUE constraint") {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
return imported, skipped, fmt.Errorf("failed to insert issue %s: %w", issue.ID, err)
|
||||
}
|
||||
|
||||
// Insert labels
|
||||
for _, label := range issue.Labels {
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO labels (issue_id, label) VALUES (?, ?)`, issue.ID, label); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to insert label %q for issue %s: %v\n", label, issue.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
imported++
|
||||
}
|
||||
|
||||
if !jsonOutput && total > 100 {
|
||||
fmt.Printf(" Importing issues: %d/%d\n", total, total)
|
||||
}
|
||||
|
||||
// Import dependencies
|
||||
printProgress("Importing dependencies...")
|
||||
for _, issue := range data.issues {
|
||||
for _, dep := range issue.Dependencies {
|
||||
var exists int
|
||||
if err := tx.QueryRowContext(ctx, "SELECT 1 FROM issues WHERE id = ?", dep.DependsOnID).Scan(&exists); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: skipping dependency %s -> %s: target issue not found\n", dep.IssueID, dep.DependsOnID)
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO dependencies (issue_id, depends_on_id, type, created_by, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE type = type
|
||||
`, dep.IssueID, dep.DependsOnID, dep.Type, dep.CreatedBy, dep.CreatedAt); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to insert dependency %s -> %s: %v\n", dep.IssueID, dep.DependsOnID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Import events (includes comments)
|
||||
printProgress("Importing events...")
|
||||
eventCount := 0
|
||||
for issueID, events := range data.eventsMap {
|
||||
for _, event := range events {
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO events (issue_id, event_type, actor, old_value, new_value, comment, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`, issueID, event.EventType, event.Actor,
|
||||
nullableStringPtr(event.OldValue), nullableStringPtr(event.NewValue),
|
||||
nullableStringPtr(event.Comment), event.CreatedAt)
|
||||
if err == nil {
|
||||
eventCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
if !jsonOutput {
|
||||
fmt.Printf(" Imported %d events\n", eventCount)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return imported, skipped, fmt.Errorf("failed to commit: %w", err)
|
||||
}
|
||||
|
||||
return imported, skipped, nil
|
||||
}
|
||||
|
||||
// Helper functions for output
|
||||
// Helper functions for output (CGO build only — used by handleToDoltMigration)
|
||||
|
||||
func exitWithError(code, message, hint string) {
|
||||
if jsonOutput {
|
||||
|
|
@ -721,55 +535,6 @@ func printFinalStatus(backend string, imported, skipped int, backupPath, newPath
|
|||
}
|
||||
}
|
||||
|
||||
// Helper functions for nullable values
|
||||
|
||||
func nullableString(s string) interface{} {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func nullableStringPtr(s *string) interface{} {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func nullableIntPtr(i *int) interface{} {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
return *i
|
||||
}
|
||||
|
||||
func nullableInt(i int) interface{} {
|
||||
if i == 0 {
|
||||
return nil
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func nullableFloat32Ptr(f *float32) interface{} {
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
return *f
|
||||
}
|
||||
|
||||
// formatJSONArray formats a string slice as JSON (matches Dolt schema expectation)
|
||||
func formatJSONArray(arr []string) string {
|
||||
if len(arr) == 0 {
|
||||
return ""
|
||||
}
|
||||
data, err := json.Marshal(arr)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// listMigrations returns registered Dolt migrations (CGO build).
|
||||
func listMigrations() []string {
|
||||
return dolt.ListMigrations()
|
||||
|
|
|
|||
272
cmd/bd/migrate_import.go
Normal file
272
cmd/bd/migrate_import.go
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage/dolt"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
"github.com/steveyegge/beads/internal/ui"
|
||||
)
|
||||
|
||||
// migrationData holds all data extracted from the source database.
|
||||
type migrationData struct {
|
||||
issues []*types.Issue
|
||||
labelsMap map[string][]string
|
||||
depsMap map[string][]*types.Dependency
|
||||
eventsMap map[string][]*types.Event
|
||||
config map[string]string
|
||||
prefix string
|
||||
issueCount int
|
||||
}
|
||||
|
||||
// findSQLiteDB looks for a SQLite .db file in the beads directory.
|
||||
// Returns the path to the first .db file found, or empty string if none.
|
||||
func findSQLiteDB(beadsDir string) string {
|
||||
// Check common names first
|
||||
for _, name := range []string{"beads.db", "issues.db"} {
|
||||
p := filepath.Join(beadsDir, name)
|
||||
if info, err := os.Stat(p); err == nil && !info.IsDir() {
|
||||
return p
|
||||
}
|
||||
}
|
||||
// Scan for any .db file
|
||||
entries, err := os.ReadDir(beadsDir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".db") &&
|
||||
!strings.Contains(entry.Name(), "backup") {
|
||||
return filepath.Join(beadsDir, entry.Name())
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseNullTime parses a time string into *time.Time. Returns nil for empty strings.
|
||||
func parseNullTime(s string) *time.Time {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.999999999Z07:00", "2006-01-02 15:04:05"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return &t
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importToDolt imports all data to Dolt, returning (imported, skipped, error)
|
||||
func importToDolt(ctx context.Context, store *dolt.DoltStore, data *migrationData) (int, int, error) {
|
||||
// Set all config values first
|
||||
for key, value := range data.config {
|
||||
if err := store.SetConfig(ctx, key, value); err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to set config %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := store.UnderlyingDB().BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
imported := 0
|
||||
skipped := 0
|
||||
seenIDs := make(map[string]bool)
|
||||
total := len(data.issues)
|
||||
|
||||
for i, issue := range data.issues {
|
||||
if !jsonOutput && total > 100 && (i+1)%100 == 0 {
|
||||
fmt.Printf(" Importing issues: %d/%d\r", i+1, total)
|
||||
}
|
||||
|
||||
if seenIDs[issue.ID] {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
seenIDs[issue.ID] = true
|
||||
|
||||
if issue.ContentHash == "" {
|
||||
issue.ContentHash = issue.ComputeContentHash()
|
||||
}
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO issues (
|
||||
id, content_hash, title, description, design, acceptance_criteria, notes,
|
||||
status, priority, issue_type, assignee, estimated_minutes,
|
||||
created_at, created_by, owner, updated_at, closed_at, external_ref,
|
||||
compaction_level, compacted_at, compacted_at_commit, original_size,
|
||||
sender, ephemeral, pinned, is_template, crystallizes,
|
||||
mol_type, work_type, quality_score, source_system, source_repo, close_reason,
|
||||
event_kind, actor, target, payload,
|
||||
await_type, await_id, timeout_ns, waiters,
|
||||
hook_bead, role_bead, agent_state, last_activity, role_type, rig,
|
||||
due_at, defer_until
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?
|
||||
)
|
||||
`,
|
||||
issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes,
|
||||
issue.Status, issue.Priority, issue.IssueType, nullableString(issue.Assignee), nullableIntPtr(issue.EstimatedMinutes),
|
||||
issue.CreatedAt, issue.CreatedBy, issue.Owner, issue.UpdatedAt, issue.ClosedAt, nullableStringPtr(issue.ExternalRef),
|
||||
issue.CompactionLevel, issue.CompactedAt, nullableStringPtr(issue.CompactedAtCommit), nullableInt(issue.OriginalSize),
|
||||
issue.Sender, issue.Ephemeral, issue.Pinned, issue.IsTemplate, issue.Crystallizes,
|
||||
issue.MolType, issue.WorkType, nullableFloat32Ptr(issue.QualityScore), issue.SourceSystem, issue.SourceRepo, issue.CloseReason,
|
||||
issue.EventKind, issue.Actor, issue.Target, issue.Payload,
|
||||
issue.AwaitType, issue.AwaitID, issue.Timeout.Nanoseconds(), formatJSONArray(issue.Waiters),
|
||||
issue.HookBead, issue.RoleBead, issue.AgentState, issue.LastActivity, issue.RoleType, issue.Rig,
|
||||
issue.DueAt, issue.DeferUntil,
|
||||
)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Duplicate entry") ||
|
||||
strings.Contains(err.Error(), "UNIQUE constraint") {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
return imported, skipped, fmt.Errorf("failed to insert issue %s: %w", issue.ID, err)
|
||||
}
|
||||
|
||||
// Insert labels
|
||||
for _, label := range issue.Labels {
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO labels (issue_id, label) VALUES (?, ?)`, issue.ID, label); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to insert label %q for issue %s: %v\n", label, issue.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
imported++
|
||||
}
|
||||
|
||||
if !jsonOutput && total > 100 {
|
||||
fmt.Printf(" Importing issues: %d/%d\n", total, total)
|
||||
}
|
||||
|
||||
// Import dependencies
|
||||
migratePrintProgress("Importing dependencies...")
|
||||
for _, issue := range data.issues {
|
||||
for _, dep := range issue.Dependencies {
|
||||
var exists int
|
||||
if err := tx.QueryRowContext(ctx, "SELECT 1 FROM issues WHERE id = ?", dep.DependsOnID).Scan(&exists); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: skipping dependency %s -> %s: target issue not found\n", dep.IssueID, dep.DependsOnID)
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO dependencies (issue_id, depends_on_id, type, created_by, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE type = type
|
||||
`, dep.IssueID, dep.DependsOnID, dep.Type, dep.CreatedBy, dep.CreatedAt); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to insert dependency %s -> %s: %v\n", dep.IssueID, dep.DependsOnID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Import events (includes comments)
|
||||
migratePrintProgress("Importing events...")
|
||||
eventCount := 0
|
||||
for issueID, events := range data.eventsMap {
|
||||
for _, event := range events {
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO events (issue_id, event_type, actor, old_value, new_value, comment, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`, issueID, event.EventType, event.Actor,
|
||||
nullableStringPtr(event.OldValue), nullableStringPtr(event.NewValue),
|
||||
nullableStringPtr(event.Comment), event.CreatedAt)
|
||||
if err == nil {
|
||||
eventCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
if !jsonOutput {
|
||||
fmt.Printf(" Imported %d events\n", eventCount)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return imported, skipped, fmt.Errorf("failed to commit: %w", err)
|
||||
}
|
||||
|
||||
return imported, skipped, nil
|
||||
}
|
||||
|
||||
// Migration output helpers
|
||||
|
||||
func migratePrintProgress(message string) {
|
||||
if !jsonOutput {
|
||||
fmt.Printf("%s\n", message)
|
||||
}
|
||||
}
|
||||
|
||||
func migratePrintSuccess(message string) {
|
||||
if !jsonOutput {
|
||||
fmt.Printf("%s\n", ui.RenderPass("✓ "+message))
|
||||
}
|
||||
}
|
||||
|
||||
func migratePrintWarning(message string) {
|
||||
if !jsonOutput {
|
||||
fmt.Printf("%s\n", ui.RenderWarn("Warning: "+message))
|
||||
}
|
||||
}
|
||||
|
||||
// Helper functions for nullable values
|
||||
|
||||
func nullableString(s string) interface{} {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func nullableStringPtr(s *string) interface{} {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func nullableIntPtr(i *int) interface{} {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
return *i
|
||||
}
|
||||
|
||||
func nullableInt(i int) interface{} {
|
||||
if i == 0 {
|
||||
return nil
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func nullableFloat32Ptr(f *float32) interface{} {
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
return *f
|
||||
}
|
||||
|
||||
// formatJSONArray formats a string slice as JSON (matches Dolt schema expectation)
|
||||
func formatJSONArray(arr []string) string {
|
||||
if len(arr) == 0 {
|
||||
return ""
|
||||
}
|
||||
data, err := json.Marshal(arr)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
585
cmd/bd/migrate_shim.go
Normal file
585
cmd/bd/migrate_shim.go
Normal file
|
|
@ -0,0 +1,585 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/beads"
|
||||
"github.com/steveyegge/beads/internal/config"
|
||||
"github.com/steveyegge/beads/internal/configfile"
|
||||
"github.com/steveyegge/beads/internal/debug"
|
||||
"github.com/steveyegge/beads/internal/storage/dolt"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// shimMigrateSQLiteToDolt performs automatic SQLite→Dolt migration using the
|
||||
// system sqlite3 CLI to export data as JSON, avoiding any CGO dependency.
|
||||
// This is the v1.0.0 upgrade path for users on SQLite who upgrade to a
|
||||
// Dolt-only bd binary.
|
||||
//
|
||||
// Steps:
|
||||
// 1. Detect beads.db (SQLite) in .beads/ with no Dolt database present
|
||||
// 2. Export all tables to JSON via the system sqlite3 CLI
|
||||
// 3. Create a new Dolt database
|
||||
// 4. Import all data into Dolt
|
||||
// 5. Rename beads.db to beads.db.migrated
|
||||
func shimMigrateSQLiteToDolt() {
|
||||
beadsDir := beads.FindBeadsDir()
|
||||
if beadsDir == "" {
|
||||
return
|
||||
}
|
||||
doShimMigrate(beadsDir)
|
||||
}
|
||||
|
||||
// doShimMigrate performs the actual migration for the given .beads directory.
|
||||
func doShimMigrate(beadsDir string) {
|
||||
// Check for SQLite database
|
||||
sqlitePath := findSQLiteDB(beadsDir)
|
||||
if sqlitePath == "" {
|
||||
return // No SQLite database, nothing to migrate
|
||||
}
|
||||
|
||||
// Skip backup/migrated files
|
||||
base := filepath.Base(sqlitePath)
|
||||
if strings.Contains(base, ".backup") || strings.Contains(base, ".migrated") {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if Dolt already exists — if so, SQLite is leftover from a prior migration
|
||||
doltPath := filepath.Join(beadsDir, "dolt")
|
||||
if _, err := os.Stat(doltPath); err == nil {
|
||||
// Dolt exists alongside SQLite. Rename the leftover SQLite file.
|
||||
migratedPath := sqlitePath + ".migrated"
|
||||
if _, err := os.Stat(migratedPath); err != nil {
|
||||
// No .migrated file yet — rename now
|
||||
if err := os.Rename(sqlitePath, migratedPath); err == nil {
|
||||
debug.Logf("shim-migrate: renamed leftover %s to %s", filepath.Base(sqlitePath), filepath.Base(migratedPath))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Verify sqlite3 CLI is available
|
||||
sqlite3Path, err := exec.LookPath("sqlite3")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: SQLite auto-migration requires the sqlite3 CLI tool\n")
|
||||
fmt.Fprintf(os.Stderr, "Hint: install sqlite3 and retry, or run 'bd migrate dolt' with a CGO-enabled build\n")
|
||||
return
|
||||
}
|
||||
debug.Logf("shim-migrate: using sqlite3 at %s", sqlite3Path)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Extract data from SQLite via CLI
|
||||
fmt.Fprintf(os.Stderr, "Migrating SQLite database to Dolt (via sqlite3 CLI)...\n")
|
||||
data, err := extractViaSQLiteCLI(ctx, sqlitePath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: SQLite auto-migration failed (extract): %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "Hint: run 'bd migrate dolt' manually, or remove %s to skip\n", base)
|
||||
return
|
||||
}
|
||||
|
||||
if data.issueCount == 0 {
|
||||
debug.Logf("shim-migrate: SQLite database is empty, skipping import")
|
||||
}
|
||||
|
||||
// Determine database name from prefix
|
||||
dbName := "beads"
|
||||
if data.prefix != "" {
|
||||
dbName = "beads_" + data.prefix
|
||||
}
|
||||
|
||||
// Load existing config for server connection settings
|
||||
doltCfg := &dolt.Config{
|
||||
Path: doltPath,
|
||||
Database: dbName,
|
||||
}
|
||||
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
|
||||
doltCfg.ServerHost = cfg.GetDoltServerHost()
|
||||
doltCfg.ServerPort = cfg.GetDoltServerPort()
|
||||
doltCfg.ServerUser = cfg.GetDoltServerUser()
|
||||
doltCfg.ServerPassword = cfg.GetDoltServerPassword()
|
||||
doltCfg.ServerTLS = cfg.GetDoltServerTLS()
|
||||
}
|
||||
|
||||
// Create Dolt store
|
||||
doltStore, err := dolt.New(ctx, doltCfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: SQLite auto-migration failed (dolt init): %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "Hint: ensure the Dolt server is running, then retry any bd command\n")
|
||||
return
|
||||
}
|
||||
|
||||
// Import data
|
||||
imported, skipped, importErr := importToDolt(ctx, doltStore, data)
|
||||
if importErr != nil {
|
||||
_ = doltStore.Close()
|
||||
_ = os.RemoveAll(doltPath)
|
||||
fmt.Fprintf(os.Stderr, "Warning: SQLite auto-migration failed (import): %v\n", importErr)
|
||||
return
|
||||
}
|
||||
|
||||
// Set sync mode
|
||||
if err := doltStore.SetConfig(ctx, "sync.mode", "dolt-native"); err != nil {
|
||||
debug.Logf("shim-migrate: failed to set sync.mode: %v", err)
|
||||
}
|
||||
|
||||
// Commit the migration
|
||||
commitMsg := fmt.Sprintf("Auto-migrate from SQLite (shim): %d issues imported", imported)
|
||||
if err := doltStore.Commit(ctx, commitMsg); err != nil {
|
||||
debug.Logf("shim-migrate: failed to create Dolt commit: %v", err)
|
||||
}
|
||||
|
||||
_ = doltStore.Close()
|
||||
|
||||
// Update metadata.json to point to Dolt
|
||||
cfg, err := configfile.Load(beadsDir)
|
||||
if err != nil || cfg == nil {
|
||||
cfg = configfile.DefaultConfig()
|
||||
}
|
||||
cfg.Backend = configfile.BackendDolt
|
||||
cfg.Database = "dolt"
|
||||
cfg.DoltDatabase = dbName
|
||||
if cfg.DoltServerPort == 0 {
|
||||
cfg.DoltServerPort = configfile.DefaultDoltServerPort
|
||||
}
|
||||
if err := cfg.Save(beadsDir); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to update metadata.json: %v\n", err)
|
||||
}
|
||||
|
||||
// Write sync.mode to config.yaml
|
||||
if err := config.SaveConfigValue("sync.mode", string(config.SyncModeDoltNative), beadsDir); err != nil {
|
||||
debug.Logf("shim-migrate: failed to write sync.mode to config.yaml: %v", err)
|
||||
}
|
||||
|
||||
// Rename SQLite file to mark migration complete
|
||||
migratedPath := sqlitePath + ".migrated"
|
||||
if err := os.Rename(sqlitePath, migratedPath); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: migration succeeded but failed to rename %s: %v\n", base, err)
|
||||
fmt.Fprintf(os.Stderr, "Hint: manually rename or remove %s\n", sqlitePath)
|
||||
}
|
||||
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "Migrated %d issues from SQLite to Dolt (%d skipped)\n", imported, skipped)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "Migrated %d issues from SQLite to Dolt\n", imported)
|
||||
}
|
||||
}
|
||||
|
||||
// extractViaSQLiteCLI extracts all data from a SQLite database by shelling
|
||||
// out to the system sqlite3 CLI. Each table is queried with .mode json and
|
||||
// the resulting JSON array is parsed into Go structs.
|
||||
func extractViaSQLiteCLI(_ context.Context, dbPath string) (*migrationData, error) {
|
||||
// Verify the file looks like a real SQLite database (check magic bytes)
|
||||
if err := verifySQLiteFile(dbPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Extract config
|
||||
configMap, err := queryJSON(dbPath, "SELECT key, value FROM config")
|
||||
if err != nil {
|
||||
// Config table might not exist in very old databases
|
||||
debug.Logf("shim-migrate: config query failed (non-fatal): %v", err)
|
||||
configMap = nil
|
||||
}
|
||||
|
||||
config := make(map[string]string)
|
||||
prefix := ""
|
||||
for _, row := range configMap {
|
||||
k, _ := row["key"].(string)
|
||||
v, _ := row["value"].(string)
|
||||
if k != "" {
|
||||
config[k] = v
|
||||
}
|
||||
if k == "issue_prefix" {
|
||||
prefix = v
|
||||
}
|
||||
}
|
||||
|
||||
// Extract issues
|
||||
issueRows, err := queryJSON(dbPath, `
|
||||
SELECT id, COALESCE(content_hash,'') as content_hash,
|
||||
COALESCE(title,'') as title, COALESCE(description,'') as description,
|
||||
COALESCE(design,'') as design, COALESCE(acceptance_criteria,'') as acceptance_criteria,
|
||||
COALESCE(notes,'') as notes,
|
||||
COALESCE(status,'') as status, COALESCE(priority,0) as priority,
|
||||
COALESCE(issue_type,'') as issue_type,
|
||||
COALESCE(assignee,'') as assignee, estimated_minutes,
|
||||
COALESCE(created_at,'') as created_at, COALESCE(created_by,'') as created_by,
|
||||
COALESCE(owner,'') as owner,
|
||||
COALESCE(updated_at,'') as updated_at, closed_at, external_ref,
|
||||
COALESCE(compaction_level,0) as compaction_level,
|
||||
COALESCE(compacted_at,'') as compacted_at, compacted_at_commit,
|
||||
COALESCE(original_size,0) as original_size,
|
||||
COALESCE(sender,'') as sender, COALESCE(ephemeral,0) as ephemeral,
|
||||
COALESCE(pinned,0) as pinned,
|
||||
COALESCE(is_template,0) as is_template, COALESCE(crystallizes,0) as crystallizes,
|
||||
COALESCE(mol_type,'') as mol_type, COALESCE(work_type,'') as work_type,
|
||||
quality_score,
|
||||
COALESCE(source_system,'') as source_system, COALESCE(source_repo,'') as source_repo,
|
||||
COALESCE(close_reason,'') as close_reason,
|
||||
COALESCE(event_kind,'') as event_kind, COALESCE(actor,'') as actor,
|
||||
COALESCE(target,'') as target, COALESCE(payload,'') as payload,
|
||||
COALESCE(await_type,'') as await_type, COALESCE(await_id,'') as await_id,
|
||||
COALESCE(timeout_ns,0) as timeout_ns, COALESCE(waiters,'') as waiters,
|
||||
COALESCE(hook_bead,'') as hook_bead, COALESCE(role_bead,'') as role_bead,
|
||||
COALESCE(agent_state,'') as agent_state,
|
||||
COALESCE(last_activity,'') as last_activity, COALESCE(role_type,'') as role_type,
|
||||
COALESCE(rig,'') as rig,
|
||||
COALESCE(due_at,'') as due_at, COALESCE(defer_until,'') as defer_until
|
||||
FROM issues`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query issues: %w", err)
|
||||
}
|
||||
|
||||
issues := make([]*types.Issue, 0, len(issueRows))
|
||||
for _, row := range issueRows {
|
||||
issue := parseIssueRow(row)
|
||||
issues = append(issues, issue)
|
||||
}
|
||||
|
||||
// Extract labels
|
||||
labelsMap := make(map[string][]string)
|
||||
labelRows, err := queryJSON(dbPath, "SELECT issue_id, label FROM labels")
|
||||
if err == nil {
|
||||
for _, row := range labelRows {
|
||||
issueID, _ := row["issue_id"].(string)
|
||||
label, _ := row["label"].(string)
|
||||
if issueID != "" && label != "" {
|
||||
labelsMap[issueID] = append(labelsMap[issueID], label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract dependencies
|
||||
depsMap := make(map[string][]*types.Dependency)
|
||||
depRows, err := queryJSON(dbPath, "SELECT issue_id, depends_on_id, COALESCE(type,'') as type, COALESCE(created_by,'') as created_by, COALESCE(created_at,'') as created_at FROM dependencies")
|
||||
if err == nil {
|
||||
for _, row := range depRows {
|
||||
dep := &types.Dependency{
|
||||
IssueID: jsonStr(row, "issue_id"),
|
||||
DependsOnID: jsonStr(row, "depends_on_id"),
|
||||
Type: types.DependencyType(jsonStr(row, "type")),
|
||||
CreatedBy: jsonStr(row, "created_by"),
|
||||
CreatedAt: jsonTime(row, "created_at"),
|
||||
}
|
||||
if dep.IssueID != "" {
|
||||
depsMap[dep.IssueID] = append(depsMap[dep.IssueID], dep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract events
|
||||
eventsMap := make(map[string][]*types.Event)
|
||||
eventRows, err := queryJSON(dbPath, "SELECT issue_id, COALESCE(event_type,'') as event_type, COALESCE(actor,'') as actor, old_value, new_value, comment, COALESCE(created_at,'') as created_at FROM events")
|
||||
if err == nil {
|
||||
for _, row := range eventRows {
|
||||
issueID := jsonStr(row, "issue_id")
|
||||
event := &types.Event{
|
||||
EventType: types.EventType(jsonStr(row, "event_type")),
|
||||
Actor: jsonStr(row, "actor"),
|
||||
CreatedAt: jsonTime(row, "created_at"),
|
||||
}
|
||||
if v := jsonNullableStr(row, "old_value"); v != nil {
|
||||
event.OldValue = v
|
||||
}
|
||||
if v := jsonNullableStr(row, "new_value"); v != nil {
|
||||
event.NewValue = v
|
||||
}
|
||||
if v := jsonNullableStr(row, "comment"); v != nil {
|
||||
event.Comment = v
|
||||
}
|
||||
if issueID != "" {
|
||||
eventsMap[issueID] = append(eventsMap[issueID], event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Assign labels and dependencies to issues
|
||||
for _, issue := range issues {
|
||||
if labels, ok := labelsMap[issue.ID]; ok {
|
||||
issue.Labels = labels
|
||||
}
|
||||
if deps, ok := depsMap[issue.ID]; ok {
|
||||
issue.Dependencies = deps
|
||||
}
|
||||
}
|
||||
|
||||
return &migrationData{
|
||||
issues: issues,
|
||||
labelsMap: labelsMap,
|
||||
depsMap: depsMap,
|
||||
eventsMap: eventsMap,
|
||||
config: config,
|
||||
prefix: prefix,
|
||||
issueCount: len(issues),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// queryJSON runs a SQL query against a SQLite database using the sqlite3 CLI
|
||||
// with JSON output mode. Returns a slice of maps representing each row.
|
||||
func queryJSON(dbPath, query string) ([]map[string]interface{}, error) {
|
||||
// Build sqlite3 command: .mode json + query
|
||||
input := fmt.Sprintf(".mode json\n%s\n", strings.TrimSpace(query))
|
||||
|
||||
cmd := exec.Command("sqlite3", "-readonly", dbPath)
|
||||
cmd.Stdin = strings.NewReader(input)
|
||||
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return nil, fmt.Errorf("sqlite3 query failed: %s", strings.TrimSpace(string(exitErr.Stderr)))
|
||||
}
|
||||
return nil, fmt.Errorf("sqlite3 query failed: %w", err)
|
||||
}
|
||||
|
||||
// Empty result
|
||||
output := strings.TrimSpace(string(out))
|
||||
if output == "" || output == "[]" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var rows []map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(output), &rows); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse sqlite3 JSON output: %w", err)
|
||||
}
|
||||
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// verifySQLiteFile checks that a file starts with the SQLite magic bytes.
|
||||
func verifySQLiteFile(path string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot open %s: %w", filepath.Base(path), err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
magic := make([]byte, 16)
|
||||
n, err := f.Read(magic)
|
||||
if err != nil || n < 16 {
|
||||
return fmt.Errorf("file too small to be a SQLite database")
|
||||
}
|
||||
|
||||
if string(magic[:15]) != "SQLite format 3" {
|
||||
return fmt.Errorf("file is not a SQLite database (bad magic bytes)")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseIssueRow converts a JSON row map into a types.Issue.
|
||||
func parseIssueRow(row map[string]interface{}) *types.Issue {
|
||||
issue := &types.Issue{
|
||||
ID: jsonStr(row, "id"),
|
||||
ContentHash: jsonStr(row, "content_hash"),
|
||||
Title: jsonStr(row, "title"),
|
||||
Description: jsonStr(row, "description"),
|
||||
Design: jsonStr(row, "design"),
|
||||
AcceptanceCriteria: jsonStr(row, "acceptance_criteria"),
|
||||
Notes: jsonStr(row, "notes"),
|
||||
Status: types.Status(jsonStr(row, "status")),
|
||||
Priority: jsonInt(row, "priority"),
|
||||
IssueType: types.IssueType(jsonStr(row, "issue_type")),
|
||||
Assignee: jsonStr(row, "assignee"),
|
||||
CreatedAt: jsonTime(row, "created_at"),
|
||||
CreatedBy: jsonStr(row, "created_by"),
|
||||
Owner: jsonStr(row, "owner"),
|
||||
UpdatedAt: jsonTime(row, "updated_at"),
|
||||
CompactionLevel: jsonInt(row, "compaction_level"),
|
||||
OriginalSize: jsonInt(row, "original_size"),
|
||||
Sender: jsonStr(row, "sender"),
|
||||
Ephemeral: jsonBool(row, "ephemeral"),
|
||||
Pinned: jsonBool(row, "pinned"),
|
||||
IsTemplate: jsonBool(row, "is_template"),
|
||||
Crystallizes: jsonBool(row, "crystallizes"),
|
||||
MolType: types.MolType(jsonStr(row, "mol_type")),
|
||||
WorkType: types.WorkType(jsonStr(row, "work_type")),
|
||||
SourceSystem: jsonStr(row, "source_system"),
|
||||
SourceRepo: jsonStr(row, "source_repo"),
|
||||
CloseReason: jsonStr(row, "close_reason"),
|
||||
EventKind: jsonStr(row, "event_kind"),
|
||||
Actor: jsonStr(row, "actor"),
|
||||
Target: jsonStr(row, "target"),
|
||||
Payload: jsonStr(row, "payload"),
|
||||
AwaitType: jsonStr(row, "await_type"),
|
||||
AwaitID: jsonStr(row, "await_id"),
|
||||
HookBead: jsonStr(row, "hook_bead"),
|
||||
RoleBead: jsonStr(row, "role_bead"),
|
||||
AgentState: types.AgentState(jsonStr(row, "agent_state")),
|
||||
RoleType: jsonStr(row, "role_type"),
|
||||
Rig: jsonStr(row, "rig"),
|
||||
}
|
||||
|
||||
// Nullable fields
|
||||
if v := jsonNullableInt(row, "estimated_minutes"); v != nil {
|
||||
issue.EstimatedMinutes = v
|
||||
}
|
||||
if v := jsonNullableStr(row, "external_ref"); v != nil {
|
||||
issue.ExternalRef = v
|
||||
}
|
||||
if v := jsonNullableStr(row, "compacted_at_commit"); v != nil {
|
||||
issue.CompactedAtCommit = v
|
||||
}
|
||||
if v := jsonNullableFloat32(row, "quality_score"); v != nil {
|
||||
issue.QualityScore = v
|
||||
}
|
||||
|
||||
// Time fields
|
||||
issue.ClosedAt = parseNullTime(jsonStr(row, "closed_at"))
|
||||
issue.CompactedAt = parseNullTime(jsonStr(row, "compacted_at"))
|
||||
issue.LastActivity = parseNullTime(jsonStr(row, "last_activity"))
|
||||
issue.DueAt = parseNullTime(jsonStr(row, "due_at"))
|
||||
issue.DeferUntil = parseNullTime(jsonStr(row, "defer_until"))
|
||||
|
||||
// Timeout duration
|
||||
issue.Timeout = time.Duration(jsonInt64(row, "timeout_ns"))
|
||||
|
||||
// Waiters
|
||||
waitersJSON := jsonStr(row, "waiters")
|
||||
if waitersJSON != "" {
|
||||
_ = json.Unmarshal([]byte(waitersJSON), &issue.Waiters)
|
||||
}
|
||||
|
||||
return issue
|
||||
}
|
||||
|
||||
// JSON row accessor helpers
|
||||
|
||||
func jsonStr(row map[string]interface{}, key string) string {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return ""
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case string:
|
||||
return val
|
||||
case float64:
|
||||
// JSON numbers come as float64
|
||||
if val == float64(int64(val)) {
|
||||
return strconv.FormatInt(int64(val), 10)
|
||||
}
|
||||
return strconv.FormatFloat(val, 'f', -1, 64)
|
||||
default:
|
||||
return fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func jsonNullableStr(row map[string]interface{}, key string) *string {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return nil
|
||||
}
|
||||
s := fmt.Sprintf("%v", v)
|
||||
return &s
|
||||
}
|
||||
|
||||
func jsonInt(row map[string]interface{}, key string) int {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return int(val)
|
||||
case string:
|
||||
i, _ := strconv.Atoi(val)
|
||||
return i
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func jsonInt64(row map[string]interface{}, key string) int64 {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return int64(val)
|
||||
case string:
|
||||
i, _ := strconv.ParseInt(val, 10, 64)
|
||||
return i
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func jsonBool(row map[string]interface{}, key string) bool {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return false
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case bool:
|
||||
return val
|
||||
case float64:
|
||||
return val != 0
|
||||
case string:
|
||||
return val == "1" || val == "true"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func jsonNullableInt(row map[string]interface{}, key string) *int {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return nil
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
i := int(val)
|
||||
return &i
|
||||
case string:
|
||||
i, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return &i
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func jsonTime(row map[string]interface{}, key string) time.Time {
|
||||
s := jsonStr(row, key)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.999999999Z07:00", "2006-01-02 15:04:05"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func jsonNullableFloat32(row map[string]interface{}, key string) *float32 {
|
||||
v, ok := row[key]
|
||||
if !ok || v == nil {
|
||||
return nil
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
f := float32(val)
|
||||
return &f
|
||||
case string:
|
||||
f, err := strconv.ParseFloat(val, 32)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
f32 := float32(f)
|
||||
return &f32
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
294
cmd/bd/migrate_shim_test.go
Normal file
294
cmd/bd/migrate_shim_test.go
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
//go:build cgo
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/configfile"
|
||||
)
|
||||
|
||||
// TestShimExtract_NoSQLite verifies the shim is a no-op when no SQLite DB exists.
|
||||
func TestShimExtract_NoSQLite(t *testing.T) {
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doShimMigrate(beadsDir)
|
||||
// Should return without doing anything — no panic, no error
|
||||
}
|
||||
|
||||
// TestShimExtract_DoltAlreadyExists verifies leftover SQLite is renamed when Dolt exists.
|
||||
func TestShimExtract_DoltAlreadyExists(t *testing.T) {
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
if err := os.WriteFile(sqlitePath, []byte("fake"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
doShimMigrate(beadsDir)
|
||||
|
||||
// beads.db should be renamed to beads.db.migrated
|
||||
if _, err := os.Stat(sqlitePath); !os.IsNotExist(err) {
|
||||
t.Error("beads.db should have been renamed")
|
||||
}
|
||||
if _, err := os.Stat(sqlitePath + ".migrated"); err != nil {
|
||||
t.Errorf("beads.db.migrated should exist: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestShimExtract_CorruptedFile verifies graceful handling of a non-SQLite file.
|
||||
func TestShimExtract_CorruptedFile(t *testing.T) {
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
if err := os.WriteFile(sqlitePath, []byte("this is not a sqlite database at all"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
doShimMigrate(beadsDir)
|
||||
|
||||
// beads.db should still exist (migration failed gracefully)
|
||||
if _, err := os.Stat(sqlitePath); err != nil {
|
||||
t.Error("beads.db should still exist after failed migration")
|
||||
}
|
||||
// dolt/ should not exist
|
||||
if _, err := os.Stat(filepath.Join(beadsDir, "dolt")); !os.IsNotExist(err) {
|
||||
t.Error("dolt/ should not exist after failed migration")
|
||||
}
|
||||
}
|
||||
|
||||
// TestShimExtract_QueryJSON verifies the sqlite3 CLI JSON extraction works.
|
||||
func TestShimExtract_QueryJSON(t *testing.T) {
|
||||
// Create a real SQLite database using the CGO driver (for test setup)
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
createTestSQLiteDB(t, sqlitePath, "shim", 3)
|
||||
|
||||
// Test queryJSON
|
||||
rows, err := queryJSON(sqlitePath, "SELECT key, value FROM config")
|
||||
if err != nil {
|
||||
t.Fatalf("queryJSON failed: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, row := range rows {
|
||||
k, _ := row["key"].(string)
|
||||
v, _ := row["value"].(string)
|
||||
if k == "issue_prefix" && v == "shim" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected config row with key=issue_prefix, value=shim; got %v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestShimExtract_ExtractViaSQLiteCLI verifies full extraction from SQLite via CLI.
|
||||
func TestShimExtract_ExtractViaSQLiteCLI(t *testing.T) {
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
createTestSQLiteDB(t, sqlitePath, "ext2", 5)
|
||||
|
||||
ctx := t.Context()
|
||||
data, err := extractViaSQLiteCLI(ctx, sqlitePath)
|
||||
if err != nil {
|
||||
t.Fatalf("extractViaSQLiteCLI failed: %v", err)
|
||||
}
|
||||
|
||||
if data.prefix != "ext2" {
|
||||
t.Errorf("expected prefix 'ext2', got %q", data.prefix)
|
||||
}
|
||||
if data.issueCount != 5 {
|
||||
t.Errorf("expected 5 issues, got %d", data.issueCount)
|
||||
}
|
||||
if len(data.issues) != 5 {
|
||||
t.Errorf("expected 5 issues in slice, got %d", len(data.issues))
|
||||
}
|
||||
|
||||
// Verify labels were loaded
|
||||
hasLabels := false
|
||||
for _, issue := range data.issues {
|
||||
if len(issue.Labels) > 0 {
|
||||
hasLabels = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasLabels {
|
||||
t.Error("expected at least one issue to have labels")
|
||||
}
|
||||
|
||||
// Verify config was loaded
|
||||
if data.config["issue_prefix"] != "ext2" {
|
||||
t.Errorf("config should contain issue_prefix=ext2, got %v", data.config)
|
||||
}
|
||||
}
|
||||
|
||||
// TestShimExtract_FullMigration does an end-to-end shim migration with a real Dolt server.
|
||||
func TestShimExtract_FullMigration(t *testing.T) {
|
||||
if testDoltServerPort == 0 {
|
||||
t.Skip("Dolt test server not available, skipping")
|
||||
}
|
||||
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write metadata.json with server config so migration can connect
|
||||
cfg := &configfile.Config{
|
||||
Database: "beads.db",
|
||||
Backend: "sqlite",
|
||||
DoltMode: configfile.DoltModeServer,
|
||||
DoltServerHost: "127.0.0.1",
|
||||
DoltServerPort: testDoltServerPort,
|
||||
}
|
||||
if err := cfg.Save(beadsDir); err != nil {
|
||||
t.Fatalf("failed to write test metadata.json: %v", err)
|
||||
}
|
||||
|
||||
// Create SQLite database with test data (using CGO driver for setup)
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
createTestSQLiteDB(t, sqlitePath, "shimmig", 3)
|
||||
|
||||
// Run shim migration
|
||||
doShimMigrate(beadsDir)
|
||||
|
||||
// Verify: beads.db renamed
|
||||
if _, err := os.Stat(sqlitePath); !os.IsNotExist(err) {
|
||||
t.Error("beads.db should have been renamed to .migrated")
|
||||
}
|
||||
if _, err := os.Stat(sqlitePath + ".migrated"); err != nil {
|
||||
t.Errorf("beads.db.migrated should exist: %v", err)
|
||||
}
|
||||
|
||||
// Verify: metadata.json updated
|
||||
updatedCfg, err := configfile.Load(beadsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load updated config: %v", err)
|
||||
}
|
||||
if updatedCfg.Backend != configfile.BackendDolt {
|
||||
t.Errorf("backend should be 'dolt', got %q", updatedCfg.Backend)
|
||||
}
|
||||
if updatedCfg.DoltDatabase != "beads_shimmig" {
|
||||
t.Errorf("dolt_database should be 'beads_shimmig', got %q", updatedCfg.DoltDatabase)
|
||||
}
|
||||
|
||||
// Verify: config.yaml has sync.mode
|
||||
configYaml := filepath.Join(beadsDir, "config.yaml")
|
||||
if data, err := os.ReadFile(configYaml); err == nil {
|
||||
if !strings.Contains(string(data), "dolt-native") {
|
||||
t.Error("config.yaml should contain sync.mode = dolt-native")
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up Dolt test database
|
||||
dropTestDatabase("beads_shimmig", testDoltServerPort)
|
||||
}
|
||||
|
||||
// TestShimExtract_VerifySQLiteFile checks magic byte validation.
|
||||
func TestShimExtract_VerifySQLiteFile(t *testing.T) {
|
||||
// Valid SQLite file
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
createTestSQLiteDB(t, sqlitePath, "verify", 1)
|
||||
|
||||
if err := verifySQLiteFile(sqlitePath); err != nil {
|
||||
t.Errorf("verifySQLiteFile should succeed for valid DB: %v", err)
|
||||
}
|
||||
|
||||
// Invalid file
|
||||
badPath := filepath.Join(beadsDir, "bad.db")
|
||||
if err := os.WriteFile(badPath, []byte("not a database file at all!!!"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifySQLiteFile(badPath); err == nil {
|
||||
t.Error("verifySQLiteFile should fail for non-SQLite file")
|
||||
}
|
||||
|
||||
// Too-small file
|
||||
tinyPath := filepath.Join(beadsDir, "tiny.db")
|
||||
if err := os.WriteFile(tinyPath, []byte("hi"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifySQLiteFile(tinyPath); err == nil {
|
||||
t.Error("verifySQLiteFile should fail for tiny file")
|
||||
}
|
||||
}
|
||||
|
||||
// TestShimExtract_ParityWithCGO verifies that the shim extraction produces
|
||||
// the same data as the CGO extractFromSQLite for the same database.
|
||||
func TestShimExtract_ParityWithCGO(t *testing.T) {
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sqlitePath := filepath.Join(beadsDir, "beads.db")
|
||||
createTestSQLiteDB(t, sqlitePath, "parity", 5)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Extract via CGO
|
||||
cgoData, err := extractFromSQLite(ctx, sqlitePath)
|
||||
if err != nil {
|
||||
t.Fatalf("extractFromSQLite failed: %v", err)
|
||||
}
|
||||
|
||||
// Extract via shim
|
||||
shimData, err := extractViaSQLiteCLI(ctx, sqlitePath)
|
||||
if err != nil {
|
||||
t.Fatalf("extractViaSQLiteCLI failed: %v", err)
|
||||
}
|
||||
|
||||
// Compare counts
|
||||
if cgoData.issueCount != shimData.issueCount {
|
||||
t.Errorf("issue count mismatch: CGO=%d, shim=%d", cgoData.issueCount, shimData.issueCount)
|
||||
}
|
||||
if cgoData.prefix != shimData.prefix {
|
||||
t.Errorf("prefix mismatch: CGO=%q, shim=%q", cgoData.prefix, shimData.prefix)
|
||||
}
|
||||
if len(cgoData.labelsMap) != len(shimData.labelsMap) {
|
||||
t.Errorf("labels map size mismatch: CGO=%d, shim=%d", len(cgoData.labelsMap), len(shimData.labelsMap))
|
||||
}
|
||||
if len(cgoData.config) != len(shimData.config) {
|
||||
t.Errorf("config map size mismatch: CGO=%d, shim=%d", len(cgoData.config), len(shimData.config))
|
||||
}
|
||||
|
||||
// Compare individual issues
|
||||
cgoIssues := make(map[string]string)
|
||||
for _, issue := range cgoData.issues {
|
||||
cgoIssues[issue.ID] = issue.Title
|
||||
}
|
||||
for _, issue := range shimData.issues {
|
||||
expected, ok := cgoIssues[issue.ID]
|
||||
if !ok {
|
||||
t.Errorf("shim has issue %s not found in CGO extraction", issue.ID)
|
||||
continue
|
||||
}
|
||||
if issue.Title != expected {
|
||||
t.Errorf("title mismatch for %s: CGO=%q, shim=%q", issue.ID, expected, issue.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue