beads/cmd/bd/export.go
2026-05-02 17:44:16 -07:00

287 lines
8.7 KiB
Go

package main
import (
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/atomicfile"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
)
var exportCmd = &cobra.Command{
Use: "export",
Short: "Export issues to JSONL format",
Long: `Export all issues to JSONL (newline-delimited JSON) format.
Each line is a complete JSON object representing one issue, including its
labels, dependencies, and comments.
This command is for issue export, migration, and interoperability. It does
not produce the JSONL backup snapshot used by 'bd backup restore'. For
supported backup/restore flows, use 'bd backup', 'bd backup export-git',
and 'bd backup restore'.
By default, exports only regular issues (excluding infrastructure beads
like agents, rigs, roles, and messages). Use --all to include everything.
Memories (from 'bd remember') are excluded by default because they may
contain sensitive agent context. Use --include-memories or --all to
include them.
EXAMPLES:
bd export # Export issues to stdout
bd export -o backup.jsonl # Export to file
bd export --include-memories # Export issues + memories
bd export --all -o full.jsonl # Include infra + templates + gates + memories
bd export --scrub -o clean.jsonl # Exclude test/pollution records`,
GroupID: "sync",
RunE: runExport,
}
var (
exportOutput string
exportAll bool
exportIncludeInfra bool
exportScrub bool
exportNoMemories bool
exportIncludeMemories bool
)
func init() {
exportCmd.Flags().StringVarP(&exportOutput, "output", "o", "", "Output file path (default: stdout)")
exportCmd.Flags().BoolVar(&exportAll, "all", false, "Include all records (infra, templates, gates, memories)")
exportCmd.Flags().BoolVar(&exportIncludeInfra, "include-infra", false, "Include infrastructure beads (agents, rigs, roles, messages)")
exportCmd.Flags().BoolVar(&exportScrub, "scrub", false, "Exclude test/pollution records")
exportCmd.Flags().BoolVar(&exportIncludeMemories, "include-memories", false, "Include persistent memories (from 'bd remember') in the export")
exportCmd.Flags().BoolVar(&exportNoMemories, "no-memories", false, "Exclude persistent memories (deprecated: now the default)")
_ = exportCmd.Flags().MarkHidden("no-memories")
rootCmd.AddCommand(exportCmd)
}
func runExport(cmd *cobra.Command, args []string) error {
ctx := rootCtx
// Determine output destination. File output uses atomic writes
// (temp file + rename) so concurrent exports and crashes never
// leave a truncated or interleaved JSONL file.
var w io.Writer
var aw *atomicfile.Writer
if exportOutput != "" {
var err error
aw, err = atomicfile.Create(exportOutput, 0o644)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer func() {
// Abort is a no-op if Close was already called.
_ = aw.Abort()
}()
w = aw
} else {
w = os.Stdout
}
// Build filter for issues table. Export all statuses (this is a backup tool).
filter := types.IssueFilter{Limit: 0}
// Exclude infra types by default (agents, rigs, roles, messages)
if !exportAll && !exportIncludeInfra {
var infraTypes []string
if store != nil {
infraSet := store.GetInfraTypes(ctx)
if len(infraSet) > 0 {
for t := range infraSet {
infraTypes = append(infraTypes, t)
}
}
}
if len(infraTypes) == 0 {
infraTypes = storage.DefaultInfraTypes()
}
for _, t := range infraTypes {
filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t))
}
}
// Exclude templates by default
if !exportAll {
isTemplate := false
filter.IsTemplate = &isTemplate
}
// Exclude ephemeral wisps by default — they are private/transient and
// must not reach git history or external integrations (GH#3649).
// --all overrides to include everything.
if !exportAll {
persistentOnly := false
filter.Ephemeral = &persistentOnly
}
issues, err := store.SearchIssues(ctx, "", filter)
if err != nil {
return fmt.Errorf("failed to search issues: %w", err)
}
// Scrub test/pollution records if requested
if exportScrub {
issues = filterOutPollution(issues)
}
if len(issues) == 0 && exportNoMemories {
if exportOutput != "" {
fmt.Fprintln(os.Stderr, "No issues to export.")
}
return nil
}
// Bulk-load relational data
issueIDs := make([]string, len(issues))
for i, issue := range issues {
issueIDs[i] = issue.ID
}
labelsMap, _ := store.GetLabelsForIssues(ctx, issueIDs)
allDeps, _ := store.GetDependencyRecordsForIssues(ctx, issueIDs)
commentsMap, _ := store.GetCommentsForIssues(ctx, issueIDs)
commentCounts, _ := store.GetCommentCounts(ctx, issueIDs)
depCounts, _ := store.GetDependencyCounts(ctx, issueIDs)
// Populate relational data on each issue
for _, issue := range issues {
issue.Labels = labelsMap[issue.ID]
issue.Dependencies = allDeps[issue.ID]
issue.Comments = commentsMap[issue.ID]
}
// Write JSONL: one JSON object per line
count := 0
for _, issue := range issues {
counts := depCounts[issue.ID]
if counts == nil {
counts = &types.DependencyCounts{}
}
// Sanitize zero-value timestamps that can't be marshaled to JSON.
// NULL datetime columns scanned as time.Time{} (year 0001) cause
// MarshalJSON to fail with "year outside of range [0,9999]". (GH#2488)
sanitizeZeroTime(issue)
record := &exportIssueRecord{
RecordType: "issue",
IssueWithCounts: &types.IssueWithCounts{
Issue: issue,
DependencyCount: counts.DependencyCount,
DependentCount: counts.DependentCount,
CommentCount: commentCounts[issue.ID],
},
}
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("failed to marshal issue %s: %w", issue.ID, err)
}
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write: %w", err)
}
if _, err := w.Write([]byte{'\n'}); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
}
count++
}
// Export memories only when explicitly requested (GH#3650).
// Memories may contain sensitive agent context and are excluded by default.
memoryCount := 0
if (exportIncludeMemories || exportAll) && !exportNoMemories {
allConfig, err := store.GetAllConfig(ctx)
if err != nil {
return fmt.Errorf("failed to read config for memories: %w", err)
}
fullPrefix := kvPrefix + memoryPrefix
// Sort keys for deterministic output order (GH#3474).
var memKeys []string
for k := range allConfig {
if strings.HasPrefix(k, fullPrefix) {
memKeys = append(memKeys, k)
}
}
sort.Strings(memKeys)
for _, k := range memKeys {
v := allConfig[k]
userKey := strings.TrimPrefix(k, fullPrefix)
record := map[string]string{
"_type": "memory",
"key": userKey,
"value": v,
}
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("failed to marshal memory %s: %w", userKey, err)
}
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write: %w", err)
}
if _, err := w.Write([]byte{'\n'}); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
}
memoryCount++
}
}
// Finalize atomic write if writing to file (fsync + rename).
if aw != nil {
if err := aw.Close(); err != nil {
return fmt.Errorf("failed to finalize export file: %w", err)
}
}
// Print summary to stderr (not stdout, to avoid mixing with JSONL)
if exportOutput != "" {
if memoryCount > 0 {
fmt.Fprintf(os.Stderr, "Exported %d issues and %d memories to %s\n", count, memoryCount, exportOutput)
} else {
fmt.Fprintf(os.Stderr, "Exported %d issues to %s\n", count, exportOutput)
}
}
return nil
}
// exportIssueRecord wraps IssueWithCounts with a _type discriminator so that
// every line in the JSONL export is self-describing. Memory lines already
// carry "_type":"memory"; this gives issue lines "_type":"issue". (GH#3271)
type exportIssueRecord struct {
RecordType string `json:"_type"`
*types.IssueWithCounts
}
// sanitizeZeroTime replaces Go zero-value time.Time fields with Unix epoch.
// NULL datetime columns in Dolt scan as time.Time{} (year 0001-01-01), which
// causes json.Marshal to fail with "year outside of range [0,9999]". (GH#2488)
func sanitizeZeroTime(issue *types.Issue) {
epoch := time.Unix(0, 0).UTC()
if issue.CreatedAt.IsZero() {
issue.CreatedAt = epoch
}
if issue.UpdatedAt.IsZero() {
issue.UpdatedAt = epoch
}
}
// filterOutPollution removes issues that look like test/pollution records.
func filterOutPollution(issues []*types.Issue) []*types.Issue {
var clean []*types.Issue
for _, issue := range issues {
if !isTestIssue(issue.Title) {
clean = append(clean, issue)
}
}
return clean
}