beads/cmd/bd/import_shared.go
Dmitry Verkhoturov ed915036c9
fix: backward-compatible JSONL import for all pre-v1.0 formats (#3021)
* fix: backward-compatible JSONL import for all pre-v1.0 formats

Three breaking changes accumulated across versions that prevent
importing old JSONL data:

1. Comment.ID changed from int64 to string in v1.0 — old JSONL
   with numeric comment IDs crashes the importer. Add custom
   UnmarshalJSON that accepts both types.

2. "wisp" field renamed to "ephemeral" in v0.38 — JSONL from
   v0.35–v0.37 silently loses the ephemeral flag. Map the old
   field name during import.

3. BondRef.proto_id renamed to source_id — old JSONL silently
   loses bond source references. Accept proto_id as alias.

Also improve auto-import error messaging: when upgrade recovery
fails, tell the user their data is safe and suggest bd init
--from-jsonl instead of silently swallowing the error.

* fix: use American English spelling for misspell linter

* test: add unit tests for backward-compatible UnmarshalJSON and legacy import

- TestCommentUnmarshalJSON: string IDs, numeric IDs (pre-v1.0),
  zero, large numbers, missing ID, invalid JSON
- TestBondRefUnmarshalJSON: source_id, proto_id (legacy),
  both fields (precedence), neither, invalid JSON
- TestImportFromLocalJSONL_LegacyFormats: numeric comment IDs
  survive import, wisp→ephemeral mapping (true/false/absent)
2026-04-04 22:34:12 -07:00

194 lines
5.9 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
"github.com/steveyegge/beads/internal/utils"
)
// ImportOptions configures import behavior.
type ImportOptions struct {
DryRun bool
SkipUpdate bool
Strict bool
RenameOnImport bool
ClearDuplicateExternalRefs bool
OrphanHandling string
DeletionIDs []string
SkipPrefixValidation bool
ProtectLocalExportIDs map[string]time.Time
}
// ImportResult describes what an import operation did.
type ImportResult struct {
Created int
Updated int
Unchanged int
Skipped int
Deleted int
Collisions int
IDMapping map[string]string
CollisionIDs []string
PrefixMismatch bool
ExpectedPrefix string
MismatchPrefixes map[string]int
SkippedDependencies []string
}
// importIssuesCore imports issues into the Dolt store.
// This is a bridge function that delegates to the Dolt store's batch creation.
func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, issues []*types.Issue, opts ImportOptions) (*ImportResult, error) {
if opts.DryRun || len(issues) == 0 {
return &ImportResult{Skipped: len(issues)}, nil
}
err := store.CreateIssuesWithFullOptions(ctx, issues, getActorWithGit(), storage.BatchCreateOptions{
OrphanHandling: storage.OrphanAllow,
SkipPrefixValidation: opts.SkipPrefixValidation,
})
if err != nil {
return nil, err
}
return &ImportResult{Created: len(issues)}, nil
}
// importLocalResult holds counts from a local JSONL import.
type importLocalResult struct {
Issues int
Memories int
}
// memoryRecord represents a memory entry in the JSONL export.
type memoryRecord struct {
Type string `json:"_type"`
Key string `json:"key"`
Value string `json:"value"`
}
// importFromLocalJSONL imports issues (and memories) from a local JSONL file on disk
// into the Dolt store. Returns the number of issues imported and any error.
// This is a convenience wrapper around importFromLocalJSONLFull.
func importFromLocalJSONL(ctx context.Context, store storage.DoltStorage, localPath string) (int, error) {
result, err := importFromLocalJSONLFull(ctx, store, localPath)
if err != nil {
return 0, err
}
return result.Issues, nil
}
// importFromLocalJSONLFull imports issues and memories from a local JSONL file.
// It detects memory records (lines with "_type":"memory") and imports them
// via SetConfig, while routing regular issue records through the normal path.
func importFromLocalJSONLFull(ctx context.Context, store storage.DoltStorage, localPath string) (*importLocalResult, error) {
//nolint:gosec // G304: path from user-provided CLI argument
data, err := os.ReadFile(localPath)
if err != nil {
return nil, fmt.Errorf("failed to read JSONL file %s: %w", localPath, err)
}
scanner := bufio.NewScanner(strings.NewReader(string(data)))
// Allow up to 64MB per line for large descriptions
scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024)
var issues []*types.Issue
var memories []memoryRecord
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
// Peek at the record to check for _type field
var peek map[string]json.RawMessage
if err := json.Unmarshal([]byte(line), &peek); err != nil {
return nil, fmt.Errorf("failed to parse JSONL line: %w", err)
}
// Check if this is a memory record
if rawType, ok := peek["_type"]; ok {
var typeStr string
if err := json.Unmarshal(rawType, &typeStr); err == nil && typeStr == "memory" {
var mem memoryRecord
if err := json.Unmarshal([]byte(line), &mem); err != nil {
return nil, fmt.Errorf("failed to parse memory record: %w", err)
}
if mem.Key != "" && mem.Value != "" {
memories = append(memories, mem)
}
continue
}
}
// Regular issue record
var issue types.Issue
if err := json.Unmarshal([]byte(line), &issue); err != nil {
return nil, fmt.Errorf("failed to parse issue from JSONL: %w", err)
}
// Skip tombstone entries: these are deleted issues exported by older
// versions (pre-v0.50) with status "tombstone" and deleted_at set.
// They are not valid for re-import since "tombstone" is not a real status.
if issue.Status == "tombstone" {
continue
}
// v0.35–v0.37 exported "wisp" (bool), renamed to "ephemeral" in v0.38+.
// map old field name so the flag is preserved on import.
if _, hasWisp := peek["wisp"]; hasWisp && !issue.Ephemeral {
var wisp bool
if err := json.Unmarshal(peek["wisp"], &wisp); err == nil && wisp {
issue.Ephemeral = true
}
}
issue.SetDefaults()
issues = append(issues, &issue)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to scan JSONL: %w", err)
}
result := &importLocalResult{}
// Import memories
for _, mem := range memories {
storageKey := kvPrefix + memoryPrefix + mem.Key
if err := store.SetConfig(ctx, storageKey, mem.Value); err != nil {
return nil, fmt.Errorf("failed to import memory %q: %w", mem.Key, err)
}
result.Memories++
}
// Import issues
if len(issues) > 0 {
// Auto-detect prefix from first issue if not already configured
configuredPrefix, err := store.GetConfig(ctx, "issue_prefix")
if err == nil && strings.TrimSpace(configuredPrefix) == "" {
firstPrefix := utils.ExtractIssuePrefix(issues[0].ID)
if firstPrefix != "" {
if err := store.SetConfig(ctx, "issue_prefix", firstPrefix); err != nil {
return nil, fmt.Errorf("failed to set issue_prefix from imported issues: %w", err)
}
}
}
opts := ImportOptions{
SkipPrefixValidation: true,
}
_, err = importIssuesCore(ctx, "", store, issues, opts)
if err != nil {
return nil, err
}
result.Issues = len(issues)
}
return result, nil
}