beads/cmd/bd/export.go
2026-05-01 16:42:47 -07:00

278 lines
8.2 KiB
Go

package main
import (
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/atomicfile"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
)
var exportCmd = &cobra.Command{
Use: "export",
Short: "Export issues to JSONL format",
Long: `Export all issues to JSONL (newline-delimited JSON) format.
Each line is a complete JSON object representing one issue, including its
labels, dependencies, and comments.
This command is for issue export, migration, and interoperability. It does
not produce the JSONL backup snapshot used by 'bd backup restore'. For
supported backup/restore flows, use 'bd backup', 'bd backup export-git',
and 'bd backup restore'.
By default, exports only regular issues (excluding infrastructure beads
like agents, rigs, roles, and messages). Use --all to include everything.
Memories (from 'bd remember') are included by default. Use --no-memories
to exclude them.
EXAMPLES:
bd export # Export issues + memories to stdout
bd export -o backup.jsonl # Export to file
bd export --no-memories # Export issues only
bd export --all -o full.jsonl # Include infra + templates + gates
bd export --scrub -o clean.jsonl # Exclude test/pollution records`,
GroupID: "sync",
RunE: runExport,
}
var (
exportOutput string
exportAll bool
exportIncludeInfra bool
exportScrub bool
exportNoMemories bool
)
func init() {
exportCmd.Flags().StringVarP(&exportOutput, "output", "o", "", "Output file path (default: stdout)")
exportCmd.Flags().BoolVar(&exportAll, "all", false, "Include all records (infra, templates, gates)")
exportCmd.Flags().BoolVar(&exportIncludeInfra, "include-infra", false, "Include infrastructure beads (agents, rigs, roles, messages)")
exportCmd.Flags().BoolVar(&exportScrub, "scrub", false, "Exclude test/pollution records")
exportCmd.Flags().BoolVar(&exportNoMemories, "no-memories", false, "Exclude persistent memories from the export")
rootCmd.AddCommand(exportCmd)
}
func runExport(cmd *cobra.Command, args []string) error {
ctx := rootCtx
// Determine output destination. File output uses atomic writes
// (temp file + rename) so concurrent exports and crashes never
// leave a truncated or interleaved JSONL file.
var w io.Writer
var aw *atomicfile.Writer
if exportOutput != "" {
var err error
aw, err = atomicfile.Create(exportOutput, 0o644)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer func() {
// Abort is a no-op if Close was already called.
_ = aw.Abort()
}()
w = aw
} else {
w = os.Stdout
}
// Build filter for issues table. Export all statuses (this is a backup tool).
filter := types.IssueFilter{Limit: 0}
// Exclude infra types by default (agents, rigs, roles, messages)
if !exportAll && !exportIncludeInfra {
var infraTypes []string
if store != nil {
infraSet := store.GetInfraTypes(ctx)
if len(infraSet) > 0 {
for t := range infraSet {
infraTypes = append(infraTypes, t)
}
}
}
if len(infraTypes) == 0 {
infraTypes = storage.DefaultInfraTypes()
}
for _, t := range infraTypes {
filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t))
}
}
// Exclude templates by default
if !exportAll {
isTemplate := false
filter.IsTemplate = &isTemplate
}
// Fetch all issues (persistent + wisps). SearchIssues with Ephemeral=nil
// already queries both the issues and wisps tables with ID-based dedup
// (see issueops/search.go). A separate Ephemeral=true query would cause
// every wisp to appear twice in the output (GH#3352).
issues, err := store.SearchIssues(ctx, "", filter)
if err != nil {
return fmt.Errorf("failed to search issues: %w", err)
}
// Scrub test/pollution records if requested
if exportScrub {
issues = filterOutPollution(issues)
}
if len(issues) == 0 && exportNoMemories {
if exportOutput != "" {
fmt.Fprintln(os.Stderr, "No issues to export.")
}
return nil
}
// Bulk-load relational data
issueIDs := make([]string, len(issues))
for i, issue := range issues {
issueIDs[i] = issue.ID
}
labelsMap, _ := store.GetLabelsForIssues(ctx, issueIDs)
allDeps, _ := store.GetDependencyRecordsForIssues(ctx, issueIDs)
commentsMap, _ := store.GetCommentsForIssues(ctx, issueIDs)
commentCounts, _ := store.GetCommentCounts(ctx, issueIDs)
depCounts, _ := store.GetDependencyCounts(ctx, issueIDs)
// Populate relational data on each issue
for _, issue := range issues {
issue.Labels = labelsMap[issue.ID]
issue.Dependencies = allDeps[issue.ID]
issue.Comments = commentsMap[issue.ID]
}
// Write JSONL: one JSON object per line
count := 0
for _, issue := range issues {
counts := depCounts[issue.ID]
if counts == nil {
counts = &types.DependencyCounts{}
}
// Sanitize zero-value timestamps that can't be marshaled to JSON.
// NULL datetime columns scanned as time.Time{} (year 0001) cause
// MarshalJSON to fail with "year outside of range [0,9999]". (GH#2488)
sanitizeZeroTime(issue)
record := &exportIssueRecord{
RecordType: "issue",
IssueWithCounts: &types.IssueWithCounts{
Issue: issue,
DependencyCount: counts.DependencyCount,
DependentCount: counts.DependentCount,
CommentCount: commentCounts[issue.ID],
},
}
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("failed to marshal issue %s: %w", issue.ID, err)
}
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write: %w", err)
}
if _, err := w.Write([]byte{'\n'}); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
}
count++
}
// Export memories if not excluded
memoryCount := 0
if !exportNoMemories {
allConfig, err := store.GetAllConfig(ctx)
if err != nil {
return fmt.Errorf("failed to read config for memories: %w", err)
}
fullPrefix := kvPrefix + memoryPrefix
// Sort keys for deterministic output order (GH#3474).
var memKeys []string
for k := range allConfig {
if strings.HasPrefix(k, fullPrefix) {
memKeys = append(memKeys, k)
}
}
sort.Strings(memKeys)
for _, k := range memKeys {
v := allConfig[k]
userKey := strings.TrimPrefix(k, fullPrefix)
record := map[string]string{
"_type": "memory",
"key": userKey,
"value": v,
}
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("failed to marshal memory %s: %w", userKey, err)
}
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write: %w", err)
}
if _, err := w.Write([]byte{'\n'}); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
}
memoryCount++
}
}
// Finalize atomic write if writing to file (fsync + rename).
if aw != nil {
if err := aw.Close(); err != nil {
return fmt.Errorf("failed to finalize export file: %w", err)
}
}
// Print summary to stderr (not stdout, to avoid mixing with JSONL)
if exportOutput != "" {
if memoryCount > 0 {
fmt.Fprintf(os.Stderr, "Exported %d issues and %d memories to %s\n", count, memoryCount, exportOutput)
} else {
fmt.Fprintf(os.Stderr, "Exported %d issues to %s\n", count, exportOutput)
}
}
return nil
}
// exportIssueRecord wraps IssueWithCounts with a _type discriminator so that
// every line in the JSONL export is self-describing. Memory lines already
// carry "_type":"memory"; this gives issue lines "_type":"issue". (GH#3271)
type exportIssueRecord struct {
RecordType string `json:"_type"`
*types.IssueWithCounts
}
// sanitizeZeroTime replaces Go zero-value time.Time fields with Unix epoch.
// NULL datetime columns in Dolt scan as time.Time{} (year 0001-01-01), which
// causes json.Marshal to fail with "year outside of range [0,9999]". (GH#2488)
func sanitizeZeroTime(issue *types.Issue) {
epoch := time.Unix(0, 0).UTC()
if issue.CreatedAt.IsZero() {
issue.CreatedAt = epoch
}
if issue.UpdatedAt.IsZero() {
issue.UpdatedAt = epoch
}
}
// filterOutPollution removes issues that look like test/pollution records.
func filterOutPollution(issues []*types.Issue) []*types.Issue {
var clean []*types.Issue
for _, issue := range issues {
if !isTestIssue(issue.Title) {
clean = append(clean, issue)
}
}
return clean
}