beads/cmd/bd/export.go
matt wilkie 39f3148fc2
docs: clarify export versus database backup (#4048)
* docs: clarify export versus database backup
* docs: remove remaining JSONL backup wording
2026-05-20 16:19:23 -07:00

288 lines
8.8 KiB
Go

package main
import (
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/atomicfile"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
)
var exportCmd = &cobra.Command{
Use: "export",
Short: "Export issues to JSONL format",
Long: `Export all issues to JSONL (newline-delimited JSON) format.
Each line is a complete JSON object representing one issue, including its
labels, dependencies, and comments.
This command is for issue export, migration, and interoperability. It exports
records from the issues table; it is not a full database backup and does not
capture Dolt branches, commit history, working-set state, or non-issue tables.
For supported full backup/restore flows, use 'bd backup init', 'bd backup sync',
and 'bd backup restore'.
By default, exports only regular issues (excluding infrastructure beads
like agents, rigs, roles, and messages). Use --all to include everything.
Memories (from 'bd remember') are excluded by default because they may
contain sensitive agent context. Use --include-memories or --all to
include them.
EXAMPLES:
bd export # Export issues to stdout
bd export -o issues.jsonl # Export issues to file
bd export --include-memories # Export issues + memories
bd export --all -o full.jsonl # Include infra + templates + gates + memories
bd export --scrub -o clean.jsonl # Exclude test/pollution records`,
GroupID: "sync",
RunE: runExport,
}
var (
exportOutput string
exportAll bool
exportIncludeInfra bool
exportScrub bool
exportNoMemories bool
exportIncludeMemories bool
)
func init() {
exportCmd.Flags().StringVarP(&exportOutput, "output", "o", "", "Output file path (default: stdout)")
exportCmd.Flags().BoolVar(&exportAll, "all", false, "Include all records (infra, templates, gates, memories)")
exportCmd.Flags().BoolVar(&exportIncludeInfra, "include-infra", false, "Include infrastructure beads (agents, rigs, roles, messages)")
exportCmd.Flags().BoolVar(&exportScrub, "scrub", false, "Exclude test/pollution records")
exportCmd.Flags().BoolVar(&exportIncludeMemories, "include-memories", false, "Include persistent memories (from 'bd remember') in the export")
exportCmd.Flags().BoolVar(&exportNoMemories, "no-memories", false, "Exclude persistent memories (deprecated: now the default)")
_ = exportCmd.Flags().MarkHidden("no-memories")
rootCmd.AddCommand(exportCmd)
}
func runExport(cmd *cobra.Command, args []string) error {
ctx := rootCtx
// Determine output destination. File output uses atomic writes
// (temp file + rename) so concurrent exports and crashes never
// leave a truncated or interleaved JSONL file.
var w io.Writer
var aw *atomicfile.Writer
if exportOutput != "" {
var err error
aw, err = atomicfile.Create(exportOutput, 0o644)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer func() {
// Abort is a no-op if Close was already called.
_ = aw.Abort()
}()
w = aw
} else {
w = os.Stdout
}
// Build filter for issues table. Export all statuses by default.
filter := types.IssueFilter{Limit: 0}
// Exclude infra types by default (agents, rigs, roles, messages)
if !exportAll && !exportIncludeInfra {
var infraTypes []string
if store != nil {
infraSet := store.GetInfraTypes(ctx)
if len(infraSet) > 0 {
for t := range infraSet {
infraTypes = append(infraTypes, t)
}
}
}
if len(infraTypes) == 0 {
infraTypes = storage.DefaultInfraTypes()
}
for _, t := range infraTypes {
filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t))
}
}
// Exclude templates by default
if !exportAll {
isTemplate := false
filter.IsTemplate = &isTemplate
}
// Exclude ephemeral wisps by default — they are private/transient and
// must not reach git history or external integrations (GH#3649).
// --all overrides to include everything.
if !exportAll {
persistentOnly := false
filter.Ephemeral = &persistentOnly
}
issues, err := store.SearchIssues(ctx, "", filter)
if err != nil {
return fmt.Errorf("failed to search issues: %w", err)
}
// Scrub test/pollution records if requested
if exportScrub {
issues = filterOutPollution(issues)
}
if len(issues) == 0 && exportNoMemories {
if exportOutput != "" {
fmt.Fprintln(os.Stderr, "No issues to export.")
}
return nil
}
// Bulk-load relational data
issueIDs := make([]string, len(issues))
for i, issue := range issues {
issueIDs[i] = issue.ID
}
labelsMap, _ := store.GetLabelsForIssues(ctx, issueIDs)
allDeps, _ := store.GetDependencyRecordsForIssues(ctx, issueIDs)
commentsMap, _ := store.GetCommentsForIssues(ctx, issueIDs)
commentCounts, _ := store.GetCommentCounts(ctx, issueIDs)
depCounts, _ := store.GetDependencyCounts(ctx, issueIDs)
// Populate relational data on each issue
for _, issue := range issues {
issue.Labels = labelsMap[issue.ID]
issue.Dependencies = allDeps[issue.ID]
issue.Comments = commentsMap[issue.ID]
}
// Write JSONL: one JSON object per line
count := 0
for _, issue := range issues {
counts := depCounts[issue.ID]
if counts == nil {
counts = &types.DependencyCounts{}
}
// Sanitize zero-value timestamps that can't be marshaled to JSON.
// NULL datetime columns scanned as time.Time{} (year 0001) cause
// MarshalJSON to fail with "year outside of range [0,9999]". (GH#2488)
sanitizeZeroTime(issue)
record := &exportIssueRecord{
RecordType: "issue",
IssueWithCounts: &types.IssueWithCounts{
Issue: issue,
DependencyCount: counts.DependencyCount,
DependentCount: counts.DependentCount,
CommentCount: commentCounts[issue.ID],
},
}
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("failed to marshal issue %s: %w", issue.ID, err)
}
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write: %w", err)
}
if _, err := w.Write([]byte{'\n'}); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
}
count++
}
// Export memories only when explicitly requested (GH#3650).
// Memories may contain sensitive agent context and are excluded by default.
memoryCount := 0
if (exportIncludeMemories || exportAll) && !exportNoMemories {
allConfig, err := store.GetAllConfig(ctx)
if err != nil {
return fmt.Errorf("failed to read config for memories: %w", err)
}
fullPrefix := kvPrefix + memoryPrefix
// Sort keys for deterministic output order (GH#3474).
var memKeys []string
for k := range allConfig {
if strings.HasPrefix(k, fullPrefix) {
memKeys = append(memKeys, k)
}
}
sort.Strings(memKeys)
for _, k := range memKeys {
v := allConfig[k]
userKey := strings.TrimPrefix(k, fullPrefix)
record := map[string]string{
"_type": "memory",
"key": userKey,
"value": v,
}
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("failed to marshal memory %s: %w", userKey, err)
}
if _, err := w.Write(data); err != nil {
return fmt.Errorf("failed to write: %w", err)
}
if _, err := w.Write([]byte{'\n'}); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
}
memoryCount++
}
}
// Finalize atomic write if writing to file (fsync + rename).
if aw != nil {
if err := aw.Close(); err != nil {
return fmt.Errorf("failed to finalize export file: %w", err)
}
}
// Print summary to stderr (not stdout, to avoid mixing with JSONL)
if exportOutput != "" {
if memoryCount > 0 {
fmt.Fprintf(os.Stderr, "Exported %d issues and %d memories to %s\n", count, memoryCount, exportOutput)
} else {
fmt.Fprintf(os.Stderr, "Exported %d issues to %s\n", count, exportOutput)
}
}
return nil
}
// exportIssueRecord wraps IssueWithCounts with a _type discriminator so that
// every line in the JSONL export is self-describing. Memory lines already
// carry "_type":"memory"; this gives issue lines "_type":"issue". (GH#3271)
type exportIssueRecord struct {
RecordType string `json:"_type"`
*types.IssueWithCounts
}
// sanitizeZeroTime replaces Go zero-value time.Time fields with Unix epoch.
// NULL datetime columns in Dolt scan as time.Time{} (year 0001-01-01), which
// causes json.Marshal to fail with "year outside of range [0,9999]". (GH#2488)
func sanitizeZeroTime(issue *types.Issue) {
epoch := time.Unix(0, 0).UTC()
if issue.CreatedAt.IsZero() {
issue.CreatedAt = epoch
}
if issue.UpdatedAt.IsZero() {
issue.UpdatedAt = epoch
}
}
// filterOutPollution removes issues that look like test/pollution records.
func filterOutPollution(issues []*types.Issue) []*types.Issue {
var clean []*types.Issue
for _, issue := range issues {
if !isTestIssue(issue.Title) {
clean = append(clean, issue)
}
}
return clean
}