* fix: backward-compatible JSONL import for all pre-v1.0 formats Three breaking changes accumulated across versions that prevent importing old JSONL data: 1. Comment.ID changed from int64 to string in v1.0 — old JSONL with numeric comment IDs crashes the importer. Add custom UnmarshalJSON that accepts both types. 2. "wisp" field renamed to "ephemeral" in v0.38 — JSONL from v0.35–v0.37 silently loses the ephemeral flag. Map the old field name during import. 3. BondRef.proto_id renamed to source_id — old JSONL silently loses bond source references. Accept proto_id as alias. Also improve auto-import error messaging: when upgrade recovery fails, tell the user their data is safe and suggest bd init --from-jsonl instead of silently swallowing the error. * fix: use American English spelling for misspell linter * test: add unit tests for backward-compatible UnmarshalJSON and legacy import - TestCommentUnmarshalJSON: string IDs, numeric IDs (pre-v1.0), zero, large numbers, missing ID, invalid JSON - TestBondRefUnmarshalJSON: source_id, proto_id (legacy), both fields (precedence), neither, invalid JSON - TestImportFromLocalJSONL_LegacyFormats: numeric comment IDs survive import, wisp→ephemeral mapping (true/false/absent)
194 lines
5.9 KiB
Go
194 lines
5.9 KiB
Go
package main
|
||
|
||
import (
|
||
"bufio"
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/steveyegge/beads/internal/storage"
|
||
"github.com/steveyegge/beads/internal/types"
|
||
"github.com/steveyegge/beads/internal/utils"
|
||
)
|
||
|
||
// ImportOptions configures import behavior.
|
||
type ImportOptions struct {
|
||
DryRun bool
|
||
SkipUpdate bool
|
||
Strict bool
|
||
RenameOnImport bool
|
||
ClearDuplicateExternalRefs bool
|
||
OrphanHandling string
|
||
DeletionIDs []string
|
||
SkipPrefixValidation bool
|
||
ProtectLocalExportIDs map[string]time.Time
|
||
}
|
||
|
||
// ImportResult describes what an import operation did.
|
||
type ImportResult struct {
|
||
Created int
|
||
Updated int
|
||
Unchanged int
|
||
Skipped int
|
||
Deleted int
|
||
Collisions int
|
||
IDMapping map[string]string
|
||
CollisionIDs []string
|
||
PrefixMismatch bool
|
||
ExpectedPrefix string
|
||
MismatchPrefixes map[string]int
|
||
SkippedDependencies []string
|
||
}
|
||
|
||
// importIssuesCore imports issues into the Dolt store.
|
||
// This is a bridge function that delegates to the Dolt store's batch creation.
|
||
func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, issues []*types.Issue, opts ImportOptions) (*ImportResult, error) {
|
||
if opts.DryRun || len(issues) == 0 {
|
||
return &ImportResult{Skipped: len(issues)}, nil
|
||
}
|
||
|
||
err := store.CreateIssuesWithFullOptions(ctx, issues, getActorWithGit(), storage.BatchCreateOptions{
|
||
OrphanHandling: storage.OrphanAllow,
|
||
SkipPrefixValidation: opts.SkipPrefixValidation,
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
return &ImportResult{Created: len(issues)}, nil
|
||
}
|
||
|
||
// importLocalResult holds counts from a local JSONL import.
|
||
type importLocalResult struct {
|
||
Issues int
|
||
Memories int
|
||
}
|
||
|
||
// memoryRecord represents a memory entry in the JSONL export.
|
||
type memoryRecord struct {
|
||
Type string `json:"_type"`
|
||
Key string `json:"key"`
|
||
Value string `json:"value"`
|
||
}
|
||
|
||
// importFromLocalJSONL imports issues (and memories) from a local JSONL file on disk
|
||
// into the Dolt store. Returns the number of issues imported and any error.
|
||
// This is a convenience wrapper around importFromLocalJSONLFull.
|
||
func importFromLocalJSONL(ctx context.Context, store storage.DoltStorage, localPath string) (int, error) {
|
||
result, err := importFromLocalJSONLFull(ctx, store, localPath)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
return result.Issues, nil
|
||
}
|
||
|
||
// importFromLocalJSONLFull imports issues and memories from a local JSONL file.
|
||
// It detects memory records (lines with "_type":"memory") and imports them
|
||
// via SetConfig, while routing regular issue records through the normal path.
|
||
func importFromLocalJSONLFull(ctx context.Context, store storage.DoltStorage, localPath string) (*importLocalResult, error) {
|
||
//nolint:gosec // G304: path from user-provided CLI argument
|
||
data, err := os.ReadFile(localPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to read JSONL file %s: %w", localPath, err)
|
||
}
|
||
|
||
scanner := bufio.NewScanner(strings.NewReader(string(data)))
|
||
// Allow up to 64MB per line for large descriptions
|
||
scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024)
|
||
var issues []*types.Issue
|
||
var memories []memoryRecord
|
||
|
||
for scanner.Scan() {
|
||
line := scanner.Text()
|
||
if line == "" {
|
||
continue
|
||
}
|
||
|
||
// Peek at the record to check for _type field
|
||
var peek map[string]json.RawMessage
|
||
if err := json.Unmarshal([]byte(line), &peek); err != nil {
|
||
return nil, fmt.Errorf("failed to parse JSONL line: %w", err)
|
||
}
|
||
|
||
// Check if this is a memory record
|
||
if rawType, ok := peek["_type"]; ok {
|
||
var typeStr string
|
||
if err := json.Unmarshal(rawType, &typeStr); err == nil && typeStr == "memory" {
|
||
var mem memoryRecord
|
||
if err := json.Unmarshal([]byte(line), &mem); err != nil {
|
||
return nil, fmt.Errorf("failed to parse memory record: %w", err)
|
||
}
|
||
if mem.Key != "" && mem.Value != "" {
|
||
memories = append(memories, mem)
|
||
}
|
||
continue
|
||
}
|
||
}
|
||
|
||
// Regular issue record
|
||
var issue types.Issue
|
||
if err := json.Unmarshal([]byte(line), &issue); err != nil {
|
||
return nil, fmt.Errorf("failed to parse issue from JSONL: %w", err)
|
||
}
|
||
// Skip tombstone entries: these are deleted issues exported by older
|
||
// versions (pre-v0.50) with status "tombstone" and deleted_at set.
|
||
// They are not valid for re-import since "tombstone" is not a real status.
|
||
if issue.Status == "tombstone" {
|
||
continue
|
||
}
|
||
|
||
// v0.35–v0.37 exported "wisp" (bool), renamed to "ephemeral" in v0.38+.
|
||
// map old field name so the flag is preserved on import.
|
||
if _, hasWisp := peek["wisp"]; hasWisp && !issue.Ephemeral {
|
||
var wisp bool
|
||
if err := json.Unmarshal(peek["wisp"], &wisp); err == nil && wisp {
|
||
issue.Ephemeral = true
|
||
}
|
||
}
|
||
|
||
issue.SetDefaults()
|
||
issues = append(issues, &issue)
|
||
}
|
||
if err := scanner.Err(); err != nil {
|
||
return nil, fmt.Errorf("failed to scan JSONL: %w", err)
|
||
}
|
||
|
||
result := &importLocalResult{}
|
||
|
||
// Import memories
|
||
for _, mem := range memories {
|
||
storageKey := kvPrefix + memoryPrefix + mem.Key
|
||
if err := store.SetConfig(ctx, storageKey, mem.Value); err != nil {
|
||
return nil, fmt.Errorf("failed to import memory %q: %w", mem.Key, err)
|
||
}
|
||
result.Memories++
|
||
}
|
||
|
||
// Import issues
|
||
if len(issues) > 0 {
|
||
// Auto-detect prefix from first issue if not already configured
|
||
configuredPrefix, err := store.GetConfig(ctx, "issue_prefix")
|
||
if err == nil && strings.TrimSpace(configuredPrefix) == "" {
|
||
firstPrefix := utils.ExtractIssuePrefix(issues[0].ID)
|
||
if firstPrefix != "" {
|
||
if err := store.SetConfig(ctx, "issue_prefix", firstPrefix); err != nil {
|
||
return nil, fmt.Errorf("failed to set issue_prefix from imported issues: %w", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
opts := ImportOptions{
|
||
SkipPrefixValidation: true,
|
||
}
|
||
_, err = importIssuesCore(ctx, "", store, issues, opts)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result.Issues = len(issues)
|
||
}
|
||
|
||
return result, nil
|
||
}
|