beads/cmd/bd/import_from_jsonl_test.go
Dmitry Verkhoturov ed915036c9
fix: backward-compatible JSONL import for all pre-v1.0 formats (#3021)
* fix: backward-compatible JSONL import for all pre-v1.0 formats

Three breaking changes accumulated across versions that prevent
importing old JSONL data:

1. Comment.ID changed from int64 to string in v1.0 — old JSONL
   with numeric comment IDs crashes the importer. Add custom
   UnmarshalJSON that accepts both types.

2. "wisp" field renamed to "ephemeral" in v0.38 — JSONL from
   v0.35–v0.37 silently loses the ephemeral flag. Map the old
   field name during import.

3. BondRef.proto_id renamed to source_id — old JSONL silently
   loses bond source references. Accept proto_id as alias.

Also improve auto-import error messaging: when upgrade recovery
fails, tell the user their data is safe and suggest bd init
--from-jsonl instead of silently swallowing the error.

* fix: use American English spelling for misspell linter

* test: add unit tests for backward-compatible UnmarshalJSON and legacy import

- TestCommentUnmarshalJSON: string IDs, numeric IDs (pre-v1.0),
  zero, large numbers, missing ID, invalid JSON
- TestBondRefUnmarshalJSON: source_id, proto_id (legacy),
  both fields (precedence), neither, invalid JSON
- TestImportFromLocalJSONL_LegacyFormats: numeric comment IDs
  survive import, wisp→ephemeral mapping (true/false/absent)
2026-04-04 22:34:12 -07:00

464 lines
17 KiB
Go

//go:build cgo
package main
import (
"context"
"os"
"path/filepath"
"testing"
)
func TestImportFromLocalJSONL(t *testing.T) {
skipIfNoDolt(t)
t.Run("imports issues from JSONL file", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
// Create a JSONL file with test issues
jsonlContent := `{"id":"test-abc123","title":"First issue","type":"bug","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-def456","title":"Second issue","type":"task","status":"open","priority":3,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("importFromLocalJSONL failed: %v", err)
}
if count != 2 {
t.Errorf("Expected 2 issues imported, got %d", count)
}
// Verify issues exist in the store
issue1, err := store.GetIssue(ctx, "test-abc123")
if err != nil {
t.Fatalf("Failed to get first issue: %v", err)
}
if issue1.Title != "First issue" {
t.Errorf("Expected title 'First issue', got %q", issue1.Title)
}
issue2, err := store.GetIssue(ctx, "test-def456")
if err != nil {
t.Fatalf("Failed to get second issue: %v", err)
}
if issue2.Title != "Second issue" {
t.Errorf("Expected title 'Second issue', got %q", issue2.Title)
}
})
t.Run("empty JSONL file imports zero issues", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(""), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("importFromLocalJSONL failed: %v", err)
}
if count != 0 {
t.Errorf("Expected 0 issues imported from empty file, got %d", count)
}
})
t.Run("nonexistent file returns error", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
ctx := context.Background()
_, err := importFromLocalJSONL(ctx, store, "/nonexistent/issues.jsonl")
if err == nil {
t.Error("Expected error for nonexistent file, got nil")
}
})
t.Run("invalid JSON returns error", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte("not valid json\n"), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
_, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err == nil {
t.Error("Expected error for invalid JSON, got nil")
}
})
t.Run("re-import with duplicate IDs succeeds via upsert", func(t *testing.T) {
// GH#2061: importing the same JSONL twice should not fail with
// "duplicate primary key" — the second import should upsert.
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
jsonlContent := `{"id":"test-dup1","title":"Original title","type":"bug","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-dup2","title":"Second issue","type":"task","status":"open","priority":3,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
// First import
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("first importFromLocalJSONL failed: %v", err)
}
if count != 2 {
t.Errorf("Expected 2 issues imported on first import, got %d", count)
}
// Second import with same IDs — should succeed (upsert), not fail
updatedContent := `{"id":"test-dup1","title":"Updated title","type":"bug","status":"closed","priority":1,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-06-01T00:00:00Z"}
{"id":"test-dup2","title":"Second issue","type":"task","status":"open","priority":3,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
if err := os.WriteFile(jsonlPath, []byte(updatedContent), 0644); err != nil {
t.Fatalf("Failed to write updated JSONL file: %v", err)
}
count2, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("second importFromLocalJSONL failed (duplicate key?): %v", err)
}
if count2 != 2 {
t.Errorf("Expected 2 issues on re-import, got %d", count2)
}
// Verify the first issue was updated (upsert, not just inserted)
issue, err := store.GetIssue(ctx, "test-dup1")
if err != nil {
t.Fatalf("Failed to get upserted issue: %v", err)
}
if issue.Title != "Updated title" {
t.Errorf("Expected title 'Updated title' after upsert, got %q", issue.Title)
}
if issue.Status != "closed" {
t.Errorf("Expected status 'closed' after upsert, got %q", issue.Status)
}
})
t.Run("child counter reconciled after JSONL import prevents overwrites", func(t *testing.T) {
// Regression test for GH#2166: bd create --parent after bd init --from-jsonl
// must not overwrite existing child issues. The child_counters table
// must be reconciled from imported hierarchical IDs.
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
// Import an epic with two existing children via JSONL
jsonlContent := `{"id":"test-epic1","title":"Epic","type":"epic","status":"open","priority":1,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-epic1.1","title":"Child 1","type":"task","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-epic1.2","title":"Child 2","type":"task","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("importFromLocalJSONL failed: %v", err)
}
if count != 3 {
t.Errorf("Expected 3 issues imported, got %d", count)
}
// Now request the next child ID for the epic — this MUST be .3, not .1
nextID, err := store.GetNextChildID(ctx, "test-epic1")
if err != nil {
t.Fatalf("GetNextChildID failed: %v", err)
}
if nextID != "test-epic1.3" {
t.Errorf("Expected next child ID 'test-epic1.3', got %q (would overwrite existing child!)", nextID)
}
// Verify original children are still intact
child1, err := store.GetIssue(ctx, "test-epic1.1")
if err != nil {
t.Fatalf("Failed to get child 1: %v", err)
}
if child1.Title != "Child 1" {
t.Errorf("Child 1 title changed unexpectedly: got %q", child1.Title)
}
child2, err := store.GetIssue(ctx, "test-epic1.2")
if err != nil {
t.Fatalf("Failed to get child 2: %v", err)
}
if child2.Title != "Child 2" {
t.Errorf("Child 2 title changed unexpectedly: got %q", child2.Title)
}
})
t.Run("skips tombstone entries during import", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
// JSONL with a mix of valid issues and tombstone entries (deleted agents from older versions)
jsonlContent := `{"id":"test-valid1","title":"Valid issue","type":"task","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-tombstone1","title":"Deleted agent","type":"agent","status":"tombstone","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-valid2","title":"Another valid issue","type":"bug","status":"open","priority":1,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-tombstone2","title":"Another deleted agent","type":"agent","status":"tombstone","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("importFromLocalJSONL failed (tombstones should be skipped, not cause errors): %v", err)
}
if count != 2 {
t.Errorf("Expected 2 issues imported (tombstones skipped), got %d", count)
}
// Verify valid issues were imported
issue1, err := store.GetIssue(ctx, "test-valid1")
if err != nil {
t.Fatalf("Failed to get valid issue 1: %v", err)
}
if issue1.Title != "Valid issue" {
t.Errorf("Expected title 'Valid issue', got %q", issue1.Title)
}
// Verify tombstone entries were NOT imported
_, err = store.GetIssue(ctx, "test-tombstone1")
if err == nil {
t.Error("Expected tombstone issue to be skipped, but it was imported")
}
})
t.Run("sets prefix from first issue when not configured", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStoreWithPrefix(t, dbPath, "") // Empty prefix
jsonlContent := `{"id":"myprefix-abc123","title":"Test issue","type":"bug","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
// Clear any existing prefix
_ = store.SetConfig(ctx, "issue_prefix", "")
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("importFromLocalJSONL failed: %v", err)
}
if count != 1 {
t.Errorf("Expected 1 issue imported, got %d", count)
}
// Verify prefix was auto-detected
prefix, err := store.GetConfig(ctx, "issue_prefix")
if err != nil {
t.Fatalf("Failed to get issue_prefix: %v", err)
}
if prefix != "myprefix" {
t.Errorf("Expected auto-detected prefix 'myprefix', got %q", prefix)
}
})
t.Run("re-import does not duplicate comments", func(t *testing.T) {
// Comments use a UUID PK (DEFAULT UUID()), so a naive INSERT would create
// duplicates on every re-import. PersistComments must deduplicate
// by checking (issue_id, author, created_at) before inserting.
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
jsonlContent := `{"id":"test-cmt1","title":"Issue with comments","type":"task","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z","comments":[{"author":"alice","text":"First comment","created_at":"2025-01-01T12:00:00Z"},{"author":"bob","text":"Second comment","created_at":"2025-01-01T13:00:00Z"}]}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
// First import
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("first import failed: %v", err)
}
if count != 1 {
t.Errorf("Expected 1 issue imported, got %d", count)
}
// Second import — same data
count2, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("second import failed: %v", err)
}
if count2 != 1 {
t.Errorf("Expected 1 issue on re-import, got %d", count2)
}
// Verify comments were NOT duplicated
issue, err := store.GetIssue(ctx, "test-cmt1")
if err != nil {
t.Fatalf("Failed to get issue: %v", err)
}
if len(issue.Comments) != 2 {
t.Errorf("Expected 2 comments after re-import, got %d (duplicates!)", len(issue.Comments))
}
})
t.Run("no_history flag survives JSONL import roundtrip", func(t *testing.T) {
// Regression test for GH#2619: ImportFromLocalJSONL must preserve no_history=true.
// NoHistory beads are stored in the wisps table with no_history=1. The issueops
// InsertIssueIntoTable function must include no_history in the INSERT or the flag
// is silently dropped and the bead becomes GC-eligible after restore.
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
// JSONL line with no_history=true (and ephemeral=false).
// This represents a NoHistory bead exported from a live database.
jsonlContent := `{"id":"test-nh1","title":"NoHistory bead","type":"task","status":"open","priority":2,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z","no_history":true}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("importFromLocalJSONL failed: %v", err)
}
if count != 1 {
t.Errorf("Expected 1 issue imported, got %d", count)
}
// Verify the bead was imported with no_history=true preserved.
issue, err := store.GetIssue(ctx, "test-nh1")
if err != nil {
t.Fatalf("Failed to get NoHistory bead after import: %v", err)
}
if issue.Title != "NoHistory bead" {
t.Errorf("Expected title 'NoHistory bead', got %q", issue.Title)
}
if !issue.NoHistory {
t.Error("no_history=true was lost during JSONL import: bead is now GC-eligible (would be incorrectly collected by wisp GC)")
}
// NoHistory beads must NOT have ephemeral=true (they're not GC-eligible)
if issue.Ephemeral {
t.Error("NoHistory bead must not be ephemeral=true after import")
}
})
}
func TestImportFromLocalJSONL_LegacyFormats(t *testing.T) {
skipIfNoDolt(t)
t.Run("numeric comment IDs from pre-v1.0", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
jsonlContent := `{"id":"test-numcmt","title":"Old comments","status":"open","priority":1,"issue_type":"task","comments":[{"id":7,"issue_id":"test-numcmt","author":"alice","text":"numeric id comment","created_at":"2025-01-01T01:00:00Z"}],"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("import with numeric comment IDs should not fail: %v", err)
}
if count != 1 {
t.Errorf("Expected 1 issue imported, got %d", count)
}
issue, err := store.GetIssue(ctx, "test-numcmt")
if err != nil {
t.Fatalf("Failed to get issue: %v", err)
}
if len(issue.Comments) != 1 {
t.Fatalf("Expected 1 comment, got %d", len(issue.Comments))
}
if issue.Comments[0].Text != "numeric id comment" {
t.Errorf("Comment text = %q, want %q", issue.Comments[0].Text, "numeric id comment")
}
})
t.Run("wisp field mapped to ephemeral", func(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "dolt")
store := newTestStore(t, dbPath)
jsonlContent := `{"id":"test-wisp1","title":"Wisp true","status":"open","priority":0,"issue_type":"task","wisp":true,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-wisp2","title":"Wisp false","status":"open","priority":0,"issue_type":"task","wisp":false,"created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
{"id":"test-wisp3","title":"No wisp field","status":"open","priority":0,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"}
`
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0644); err != nil {
t.Fatalf("Failed to write JSONL file: %v", err)
}
ctx := context.Background()
count, err := importFromLocalJSONL(ctx, store, jsonlPath)
if err != nil {
t.Fatalf("import with wisp field should not fail: %v", err)
}
if count != 3 {
t.Errorf("Expected 3 issues imported, got %d", count)
}
issue1, err := store.GetIssue(ctx, "test-wisp1")
if err != nil {
t.Fatalf("Failed to get wisp=true issue: %v", err)
}
if !issue1.Ephemeral {
t.Error("wisp=true should map to ephemeral=true")
}
issue2, err := store.GetIssue(ctx, "test-wisp2")
if err != nil {
t.Fatalf("Failed to get wisp=false issue: %v", err)
}
if issue2.Ephemeral {
t.Error("wisp=false should not set ephemeral=true")
}
issue3, err := store.GetIssue(ctx, "test-wisp3")
if err != nil {
t.Fatalf("Failed to get no-wisp issue: %v", err)
}
if issue3.Ephemeral {
t.Error("missing wisp field should not set ephemeral=true")
}
})
}