diff --git a/cmd/bd/bootstrap_embedded_test.go b/cmd/bd/bootstrap_embedded_test.go index afbb22e34..b700a8183 100644 --- a/cmd/bd/bootstrap_embedded_test.go +++ b/cmd/bd/bootstrap_embedded_test.go @@ -68,18 +68,18 @@ func TestBootstrapNoWorkspace(t *testing.T) { t.Fatalf("expected JSON object in output, got: %s", out) } - var payload map[string]string + var payload map[string]interface{} if err := json.Unmarshal([]byte(s[start:]), &payload); err != nil { t.Fatalf("parse bootstrap JSON: %v\n%s", err, s) } - if payload["action"] != "none" { - t.Fatalf("action = %q, want %q", payload["action"], "none") + if action, _ := payload["action"].(string); action != "none" { + t.Fatalf("action = %q, want %q", action, "none") } - if payload["reason"] != activeWorkspaceNotFoundError() { - t.Fatalf("reason = %q, want %q", payload["reason"], activeWorkspaceNotFoundError()) + if reason, _ := payload["reason"].(string); reason != activeWorkspaceNotFoundError() { + t.Fatalf("reason = %q, want %q", reason, activeWorkspaceNotFoundError()) } - if !strings.Contains(payload["suggestion"], "bd where") { - t.Fatalf("suggestion should mention bd where, got: %q", payload["suggestion"]) + if suggestion, _ := payload["suggestion"].(string); !strings.Contains(suggestion, "bd where") { + t.Fatalf("suggestion should mention bd where, got: %q", suggestion) } }) } diff --git a/cmd/bd/config_embedded_test.go b/cmd/bd/config_embedded_test.go index 818c329f0..203ff48f4 100644 --- a/cmd/bd/config_embedded_test.go +++ b/cmd/bd/config_embedded_test.go @@ -55,10 +55,19 @@ func bdConfigListJSON(t *testing.T, bd, dir string) map[string]string { if start < 0 { t.Fatalf("no JSON object in config list output: %s", s) } - var m map[string]string - if err := json.Unmarshal([]byte(s[start:]), &m); err != nil { + var raw map[string]interface{} + if err := json.Unmarshal([]byte(s[start:]), &raw); err != nil { t.Fatalf("parse config list JSON: %v\n%s", err, s) } + m := make(map[string]string, len(raw)) + for k, v := range raw { + if k == "schema_version" { + continue + } + if sv, ok := v.(string); ok { + m[k] = sv + } + } return m } diff --git a/cmd/bd/config_nodb_embedded_test.go b/cmd/bd/config_nodb_embedded_test.go index a8adfcd50..a665362ff 100644 --- a/cmd/bd/config_nodb_embedded_test.go +++ b/cmd/bd/config_nodb_embedded_test.go @@ -152,14 +152,14 @@ func TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout(t *testing.T) { t.Fatalf("expected JSON error on stdout only, got stderr:\n%s", stderr) } - var payload map[string]string + var payload map[string]interface{} if err := json.Unmarshal([]byte(stdout), &payload); err != nil { t.Fatalf("parse config validate --json output: %v\nstdout:\n%s", err, stdout) } - if payload["error"] != activeWorkspaceNotFoundError() { - t.Fatalf("error = %q, want %q", payload["error"], activeWorkspaceNotFoundError()) + if errField, _ := payload["error"].(string); errField != activeWorkspaceNotFoundError() { + t.Fatalf("error = %q, want %q", errField, activeWorkspaceNotFoundError()) } - if !strings.Contains(payload["hint"], "bd where") { - t.Fatalf("expected hint to mention bd where, got %q", payload["hint"]) + if hint, _ := payload["hint"].(string); !strings.Contains(hint, "bd where") { + t.Fatalf("expected hint to mention bd where, got %q", hint) } } diff --git a/cmd/bd/errors.go b/cmd/bd/errors.go index d8d848e2b..6a0aa0665 100644 --- a/cmd/bd/errors.go +++ b/cmd/bd/errors.go @@ -38,6 +38,37 @@ func workspaceDiagHint(includeWhere bool) string { return "check BEADS_DIR/worktree setup, run 'bd doctor' to diagnose, or run 'bd init' to create a new database" } +// jsonStderrError writes a structured JSON error to stderr when --json is active. +// All JSON errors include schema_version for consumer compatibility. +func jsonStderrError(message, hint string) { + obj := map[string]interface{}{ + "schema_version": JSONSchemaVersion, + "error": message, + } + if hint != "" { + obj["hint"] = hint + } + encoder := json.NewEncoder(os.Stderr) + encoder.SetIndent("", " ") + _ = encoder.Encode(obj) +} + +// jsonStdoutError writes a structured JSON error to stdout when --json is active. +// Used by FatalErrorRespectJSON and FatalErrorWithHintRespectJSON where +// callers expect errors on stdout (e.g., bd show nonexistent-id --json). +func jsonStdoutError(message, hint string) { + obj := map[string]interface{}{ + "schema_version": JSONSchemaVersion, + "error": message, + } + if hint != "" { + obj["hint"] = hint + } + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + _ = encoder.Encode(obj) +} + // FatalError writes an error message to stderr and exits with code 1. // Use this for fatal errors that prevent the command from completing. // @@ -54,8 +85,7 @@ func workspaceDiagHint(includeWhere bool) string { func FatalError(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) if jsonOutput { - data, _ := json.MarshalIndent(map[string]string{"error": msg}, "", " ") - fmt.Fprintln(os.Stderr, string(data)) + jsonStderrError(msg, "") } else { fmt.Fprintf(os.Stderr, "Error: %s\n", msg) } @@ -76,8 +106,7 @@ func FatalError(format string, args ...interface{}) { func FatalErrorRespectJSON(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) if jsonOutput { - data, _ := json.MarshalIndent(map[string]string{"error": msg}, "", " ") // json.MarshalIndent on simple maps does not fail in practice - fmt.Println(string(data)) + jsonStdoutError(msg, "") } else { fmt.Fprintf(os.Stderr, "Error: %s\n", msg) } @@ -88,7 +117,7 @@ func FatalErrorRespectJSON(format string, args ...interface{}) { // If --json is set, emits structured JSON to stdout so callers can parse it. func FatalErrorWithHintRespectJSON(message, hint string) { if jsonOutput { - outputJSON(map[string]string{"error": message, "hint": hint}) + jsonStdoutError(message, hint) } else { fmt.Fprintf(os.Stderr, "Error: %s\n", message) fmt.Fprintf(os.Stderr, "Hint: %s\n", hint) @@ -104,8 +133,7 @@ func FatalErrorWithHintRespectJSON(message, hint string) { // FatalErrorWithHint("database not found", "Run 'bd init' to create a database") func FatalErrorWithHint(message, hint string) { if jsonOutput { - data, _ := json.MarshalIndent(map[string]string{"error": message, "hint": hint}, "", " ") - fmt.Fprintln(os.Stderr, string(data)) + jsonStderrError(message, hint) } else { fmt.Fprintf(os.Stderr, "Error: %s\n", message) fmt.Fprintf(os.Stderr, "Hint: %s\n", hint) diff --git a/cmd/bd/errors_test.go b/cmd/bd/errors_test.go new file mode 100644 index 000000000..c04a2dd84 --- /dev/null +++ b/cmd/bd/errors_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func TestJsonStderrError_StructuredOutput(t *testing.T) { + tests := []struct { + name string + message string + hint string + }{ + {"message_only", "database not found", ""}, + {"message_with_hint", "database not found", "Run 'bd init' to create one"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := map[string]interface{}{ + "schema_version": JSONSchemaVersion, + "error": tt.message, + } + if tt.hint != "" { + obj["hint"] = tt.hint + } + + data, err := json.Marshal(obj) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var parsed map[string]interface{} + if err := json.Unmarshal(data, &parsed); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if parsed["schema_version"] != float64(JSONSchemaVersion) { + t.Errorf("schema_version = %v, want %d", parsed["schema_version"], JSONSchemaVersion) + } + if parsed["error"] != tt.message { + t.Errorf("error = %v, want %s", parsed["error"], tt.message) + } + if tt.hint != "" { + if parsed["hint"] != tt.hint { + t.Errorf("hint = %v, want %s", parsed["hint"], tt.hint) + } + } else { + if _, ok := parsed["hint"]; ok { + t.Errorf("hint should not be present when empty") + } + } + }) + } +} diff --git a/cmd/bd/import.go b/cmd/bd/import.go index e584f4212..dfb8cd399 100644 --- a/cmd/bd/import.go +++ b/cmd/bd/import.go @@ -1,56 +1,74 @@ package main import ( + "bufio" + "context" + "encoding/json" "fmt" + "io" "os" "path/filepath" + "strings" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" ) var importCmd = &cobra.Command{ - Use: "import [file]", - Short: "Import issues from a JSONL file into the database", + Use: "import [file|-]", + Short: "Import issues from a JSONL file or stdin into the database", Long: `Import issues from a JSONL file (newline-delimited JSON) into the database. If no file is specified, imports from .beads/issues.jsonl (the git-tracked -export). This is the incremental counterpart to 'bd export': new issues are -created and existing issues are updated (upsert semantics). +export). Use "-" to read from stdin. This is the incremental counterpart to +'bd export': new issues are created and existing issues are updated (upsert +semantics). Memory records (lines with "_type":"memory") are automatically detected and imported as persistent memories (equivalent to 'bd remember'). This makes 'bd export | bd import' a full round-trip for both issues and memories. -This command makes the git-tracked JSONL portable again — after 'git pull' -brings new issues, 'bd import' loads them into the local Dolt database. +Each JSONL line should map to an issue with at minimum "title". Optional +fields: description, issue_type (type), priority, acceptance_criteria. EXAMPLES: bd import # Import from .beads/issues.jsonl bd import backup.jsonl # Import from a specific file - bd import --dry-run # Show what would be imported`, + bd import - # Read JSONL from stdin + cat issues.jsonl | bd import - # Pipe JSONL from another tool + bd import --dry-run # Show what would be imported + bd import --dedup # Skip issues with duplicate titles + bd import --json # Structured output with created IDs`, GroupID: "sync", RunE: runImport, } var ( importDryRun bool + importDedup bool ) func init() { importCmd.Flags().BoolVar(&importDryRun, "dry-run", false, "Show what would be imported without importing") + importCmd.Flags().BoolVar(&importDedup, "dedup", false, "Skip lines whose title matches an existing open issue") rootCmd.AddCommand(importCmd) } func runImport(cmd *cobra.Command, args []string) error { ctx := rootCtx + fromStdin := len(args) > 0 && args[0] == "-" + + if fromStdin { + return runImportFromReader(ctx, os.Stdin, "stdin") + } // Determine source file var jsonlPath string if len(args) > 0 { jsonlPath = args[0] } else { - // Default: .beads/issues.jsonl (or .beads/global-issues.jsonl with --global) beadsDir := beads.FindBeadsDir() if beadsDir == "" { return fmt.Errorf("%s — %s", activeWorkspaceNotFoundError(), diagHint()) @@ -62,44 +80,194 @@ func runImport(cmd *cobra.Command, args []string) error { } } - // Check file exists info, err := os.Stat(jsonlPath) if err != nil { return fmt.Errorf("cannot read %s: %w", jsonlPath, err) } if info.Size() == 0 { + if jsonOutput { + outputJSON(importResultJSON{Source: jsonlPath}) + return nil + } fmt.Fprintf(os.Stderr, "Empty file: %s\n", jsonlPath) return nil } - if importDryRun { - fmt.Fprintf(os.Stderr, "Would import from: %s (%d bytes)\n", jsonlPath, info.Size()) - return nil + f, err := os.Open(jsonlPath) //nolint:gosec // G304: CLI argument + if err != nil { + return fmt.Errorf("cannot open %s: %w", jsonlPath, err) } + defer f.Close() - // store is the global Dolt store, opened by main.go's PersistentPreRunE + return runImportFromReader(ctx, f, jsonlPath) +} + +type importResultJSON struct { + Source string `json:"source"` + Created int `json:"created"` + Skipped int `json:"skipped"` + DedupHits int `json:"dedup_skipped,omitempty"` + Memories int `json:"memories,omitempty"` + IDs []string `json:"ids,omitempty"` + DryRun bool `json:"dry_run,omitempty"` +} + +func runImportFromReader(ctx context.Context, r io.Reader, source string) error { if store == nil { return fmt.Errorf("no database — run 'bd init' or 'bd bootstrap' first") } - result, err := importFromLocalJSONLFull(ctx, store, jsonlPath) - if err != nil { - return fmt.Errorf("import failed: %w", err) + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024) + + var issues []*types.Issue + var memories []memoryRecord + + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + + var peek map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &peek); err != nil { + return fmt.Errorf("failed to parse JSONL line: %w", err) + } + + if rawType, ok := peek["_type"]; ok { + var typeStr string + if err := json.Unmarshal(rawType, &typeStr); err == nil && typeStr == "memory" { + var mem memoryRecord + if err := json.Unmarshal([]byte(line), &mem); err != nil { + return fmt.Errorf("failed to parse memory record: %w", err) + } + if mem.Key != "" && mem.Value != "" { + memories = append(memories, mem) + } + continue + } + } + + var issue types.Issue + if err := json.Unmarshal([]byte(line), &issue); err != nil { + return fmt.Errorf("failed to parse issue from JSONL: %w", err) + } + if issue.Status == "tombstone" { + continue + } + if _, hasWisp := peek["wisp"]; hasWisp && !issue.Ephemeral { + var wisp bool + if err := json.Unmarshal(peek["wisp"], &wisp); err == nil && wisp { + issue.Ephemeral = true + } + } + issue.SetDefaults() + issues = append(issues, &issue) + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("failed to scan JSONL: %w", err) } - commitMsg := fmt.Sprintf("bd import: %d issues", result.Issues) + // Dedup: skip issues whose title matches an existing open issue + dedupHits := 0 + if importDedup && len(issues) > 0 { + issues, dedupHits = filterDuplicatesByTitle(ctx, store, issues) + } + + result := importResultJSON{ + Source: source, + DedupHits: dedupHits, + DryRun: importDryRun, + } + + if importDryRun { + result.Created = len(issues) + result.Memories = len(memories) + result.Skipped = dedupHits + if jsonOutput { + outputJSON(result) + return nil + } + fmt.Fprintf(os.Stderr, "Would import %d issues and %d memories from %s", len(issues), len(memories), source) + if dedupHits > 0 { + fmt.Fprintf(os.Stderr, " (%d duplicates skipped)", dedupHits) + } + fmt.Fprintln(os.Stderr) + return nil + } + + // Import memories + for _, mem := range memories { + storageKey := kvPrefix + memoryPrefix + mem.Key + if err := store.SetConfig(ctx, storageKey, mem.Value); err != nil { + return fmt.Errorf("failed to import memory %q: %w", mem.Key, err) + } + result.Memories++ + } + + // Import issues + if len(issues) > 0 { + opts := ImportOptions{SkipPrefixValidation: true} + importResult, err := importIssuesCore(ctx, "", store, issues, opts) + if err != nil { + return fmt.Errorf("import failed: %w", err) + } + result.Created = importResult.Created + result.Skipped += importResult.Skipped + for _, issue := range issues { + result.IDs = append(result.IDs, issue.ID) + } + } + + // Commit + commitMsg := fmt.Sprintf("bd import: %d issues", result.Created) if result.Memories > 0 { commitMsg += fmt.Sprintf(", %d memories", result.Memories) } - commitMsg += fmt.Sprintf(" from %s", filepath.Base(jsonlPath)) + commitMsg += fmt.Sprintf(" from %s", filepath.Base(source)) if err := store.Commit(ctx, commitMsg); err != nil { return fmt.Errorf("commit: %w", err) } - if result.Memories > 0 { - fmt.Fprintf(os.Stderr, "Imported %d issues and %d memories from %s\n", result.Issues, result.Memories, jsonlPath) - } else { - fmt.Fprintf(os.Stderr, "Imported %d issues from %s\n", result.Issues, jsonlPath) + if jsonOutput { + outputJSON(result) + return nil } + + fmt.Fprintf(os.Stderr, "Imported %d issues", result.Created) + if result.Memories > 0 { + fmt.Fprintf(os.Stderr, " and %d memories", result.Memories) + } + fmt.Fprintf(os.Stderr, " from %s", source) + if dedupHits > 0 { + fmt.Fprintf(os.Stderr, " (%d duplicates skipped)", dedupHits) + } + fmt.Fprintln(os.Stderr) return nil } + +// filterDuplicatesByTitle removes issues whose title matches an existing open issue. +func filterDuplicatesByTitle(ctx context.Context, st storage.DoltStorage, issues []*types.Issue) ([]*types.Issue, int) { + existing, err := st.SearchIssues(ctx, "", types.IssueFilter{}) + if err != nil { + return issues, 0 + } + + titleSet := make(map[string]bool, len(existing)) + for _, issue := range existing { + if issue.Status != types.StatusClosed { + titleSet[strings.ToLower(issue.Title)] = true + } + } + + var kept []*types.Issue + skipped := 0 + for _, issue := range issues { + if titleSet[strings.ToLower(issue.Title)] { + skipped++ + continue + } + kept = append(kept, issue) + } + return kept, skipped +} diff --git a/cmd/bd/kv_embedded_test.go b/cmd/bd/kv_embedded_test.go index 2c686e9ab..4954131c9 100644 --- a/cmd/bd/kv_embedded_test.go +++ b/cmd/bd/kv_embedded_test.go @@ -53,13 +53,21 @@ func bdKVListJSON(t *testing.T, bd, dir string) map[string]string { s := strings.TrimSpace(string(out)) start := strings.Index(s, "{") if start < 0 { - // Empty KV store may return empty object or nothing return map[string]string{} } - var m map[string]string - if err := json.Unmarshal([]byte(s[start:]), &m); err != nil { + var raw map[string]interface{} + if err := json.Unmarshal([]byte(s[start:]), &raw); err != nil { t.Fatalf("parse kv list JSON: %v\n%s", err, s) } + m := make(map[string]string, len(raw)) + for k, v := range raw { + if k == "schema_version" { + continue + } + if sv, ok := v.(string); ok { + m[k] = sv + } + } return m } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 44252d300..db4eaec69 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -110,6 +110,7 @@ var readOnlyCommands = map[string]bool{ "duplicates": true, "comments": true, // list comments (not add) "current": true, // bd sync mode current + "ping": true, "backup": true, // reads from Dolt, writes only to .beads/backup/ "export": true, // reads from Dolt, writes JSONL to file/stdout } diff --git a/cmd/bd/output.go b/cmd/bd/output.go index 4c7ffac9b..6c18d4e9b 100644 --- a/cmd/bd/output.go +++ b/cmd/bd/output.go @@ -3,10 +3,32 @@ package main import ( "encoding/json" "os" + "reflect" ) +// JSONSchemaVersion is the current version of the bd JSON output schema. +// Consumers can check this field to detect format changes. Bump when +// fields are added, renamed, or removed from any --json output. +const JSONSchemaVersion = 1 + // outputJSON outputs data as pretty-printed JSON to stdout. +// +// Object input: schema_version is injected as a top-level field so consumers +// (Jawnt MCP, BeadsX, gt mail, sync scripts) can detect format changes. +// Array/slice input: output as-is (no envelope wrapping) to preserve +// backwards compatibility with existing consumers that parse raw arrays. func outputJSON(v interface{}) { + wrapped := wrapWithSchemaVersion(v) + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(wrapped); err != nil { + FatalError("encoding JSON: %v", err) + } +} + +// outputJSONRaw outputs data without schema_version wrapping. +// Use for internal/machine-only output that should not be versioned. +func outputJSONRaw(v interface{}) { encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") if err := encoder.Encode(v); err != nil { @@ -14,14 +36,48 @@ func outputJSON(v interface{}) { } } +// wrapWithSchemaVersion adds schema_version to object output. Arrays +// and slices are returned unchanged to preserve backwards compatibility +// with existing consumers that parse raw JSON arrays. +func wrapWithSchemaVersion(v interface{}) interface{} { + if v == nil { + return map[string]interface{}{"schema_version": JSONSchemaVersion} + } + + rv := reflect.ValueOf(v) + for rv.Kind() == reflect.Ptr { + rv = rv.Elem() + } + + // Arrays/slices: return as-is (no envelope) for backwards compat. + if rv.Kind() == reflect.Slice || rv.Kind() == reflect.Array { + return v + } + + // Objects: inject schema_version as a top-level field. + data, err := json.Marshal(v) + if err != nil { + return v + } + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + return v + } + m["schema_version"] = JSONSchemaVersion + return m +} + // outputJSONError outputs an error as JSON to stderr and exits with code 1. func outputJSONError(err error, code string) { - errObj := map[string]string{"error": err.Error()} + errObj := map[string]interface{}{ + "error": err.Error(), + "schema_version": JSONSchemaVersion, + } if code != "" { errObj["code"] = code } encoder := json.NewEncoder(os.Stderr) encoder.SetIndent("", " ") - _ = encoder.Encode(errObj) // Best effort: if JSON encoding fails, error is already printed to stderr + _ = encoder.Encode(errObj) os.Exit(1) } diff --git a/cmd/bd/output_test.go b/cmd/bd/output_test.go index 5974e8132..7a6032a7c 100644 --- a/cmd/bd/output_test.go +++ b/cmd/bd/output_test.go @@ -1,97 +1,90 @@ package main import ( - "bytes" "encoding/json" - "io" - "os" "testing" ) -func TestOutputJSON(t *testing.T) { - // Capture stdout - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w +func TestWrapWithSchemaVersion_Object(t *testing.T) { + input := map[string]string{"id": "beads-123", "title": "Test"} + result := wrapWithSchemaVersion(input) - // Test data - testData := map[string]interface{}{ - "id": "bd-1", - "title": "Test Issue", - "count": 42, + m, ok := result.(map[string]interface{}) + if !ok { + t.Fatalf("expected map[string]interface{}, got %T", result) } - - // Call outputJSON - outputJSON(testData) - - // Restore stdout - w.Close() - os.Stdout = oldStdout - - // Read output - var buf bytes.Buffer - io.Copy(&buf, r) - output := buf.String() - - // Verify it's valid JSON - var result map[string]interface{} - err := json.Unmarshal([]byte(output), &result) - if err != nil { - t.Fatalf("outputJSON did not produce valid JSON: %v", err) + if m["schema_version"] != JSONSchemaVersion { + t.Errorf("schema_version = %v, want %d", m["schema_version"], JSONSchemaVersion) } - - // Verify content - if result["id"] != "bd-1" { - t.Errorf("Expected id 'bd-1', got '%v'", result["id"]) - } - if result["title"] != "Test Issue" { - t.Errorf("Expected title 'Test Issue', got '%v'", result["title"]) - } - // Note: JSON numbers are float64 - if result["count"] != float64(42) { - t.Errorf("Expected count 42, got %v", result["count"]) + if m["id"] != "beads-123" { + t.Errorf("id = %v, want beads-123", m["id"]) } } -func TestOutputJSONArray(t *testing.T) { - // Capture stdout - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - // Test data - array of issues - testData := []map[string]string{ - {"id": "bd-1", "title": "First"}, - {"id": "bd-2", "title": "Second"}, +func TestWrapWithSchemaVersion_Struct(t *testing.T) { + type issue struct { + ID string `json:"id"` + Title string `json:"title"` } + input := &issue{ID: "beads-456", Title: "Struct test"} + result := wrapWithSchemaVersion(input) - // Call outputJSON - outputJSON(testData) - - // Restore stdout - w.Close() - os.Stdout = oldStdout - - // Read output - var buf bytes.Buffer - io.Copy(&buf, r) - output := buf.String() - - // Verify it's valid JSON array - var result []map[string]string - err := json.Unmarshal([]byte(output), &result) - if err != nil { - t.Fatalf("outputJSON did not produce valid JSON array: %v", err) + m, ok := result.(map[string]interface{}) + if !ok { + t.Fatalf("expected map[string]interface{}, got %T", result) } - - if len(result) != 2 { - t.Fatalf("Expected 2 items, got %d", len(result)) + if m["schema_version"] != JSONSchemaVersion { + t.Errorf("schema_version = %v, want %d", m["schema_version"], JSONSchemaVersion) + } + if m["id"] != "beads-456" { + t.Errorf("id = %v, want beads-456", m["id"]) } } -// Tests for printCollisionReport and printRemappingReport were removed -// These functions no longer exist after refactoring to shared importIssuesCore (bd-157) +func TestWrapWithSchemaVersion_Slice(t *testing.T) { + input := []string{"a", "b", "c"} + result := wrapWithSchemaVersion(input) -// Note: createIssuesFromMarkdown is tested via cmd/bd/markdown_test.go which has -// comprehensive tests for the markdown parsing functionality. We don't duplicate -// those tests here since they require full DB setup. + // Arrays pass through unchanged for backwards compatibility. + arr, ok := result.([]string) + if !ok { + t.Fatalf("expected []string (passthrough), got %T", result) + } + if len(arr) != 3 { + t.Errorf("slice length = %d, want 3", len(arr)) + } +} + +func TestWrapWithSchemaVersion_Nil(t *testing.T) { + result := wrapWithSchemaVersion(nil) + m, ok := result.(map[string]interface{}) + if !ok { + t.Fatalf("expected map[string]interface{}, got %T", result) + } + if m["schema_version"] != JSONSchemaVersion { + t.Errorf("schema_version = %v, want %d", m["schema_version"], JSONSchemaVersion) + } +} + +func TestWrapWithSchemaVersion_RoundTrip(t *testing.T) { + input := map[string]interface{}{"count": 42, "name": "test"} + result := wrapWithSchemaVersion(input) + + data, err := json.Marshal(result) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var parsed map[string]interface{} + if err := json.Unmarshal(data, &parsed); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + sv, ok := parsed["schema_version"] + if !ok { + t.Fatal("schema_version missing after round-trip") + } + if sv != float64(JSONSchemaVersion) { + t.Errorf("schema_version = %v, want %v", sv, float64(JSONSchemaVersion)) + } +} diff --git a/cmd/bd/ping.go b/cmd/bd/ping.go new file mode 100644 index 000000000..0ee62c413 --- /dev/null +++ b/cmd/bd/ping.go @@ -0,0 +1,94 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" +) + +var pingCmd = &cobra.Command{ + Use: "ping", + GroupID: "maint", + Short: "Check database connectivity", + Long: `Lightweight health check that confirms bd can reach its database. + +Steps: + 1. Resolve the .beads workspace + 2. Open the store (embedded or server) + 3. Run a trivial query (issue count) + 4. Report timing + +Exit 0 on success, exit 1 on failure. + +Examples: + bd ping # Quick connectivity check + bd ping --json # Structured output for automation`, + Run: func(cmd *cobra.Command, args []string) { + start := time.Now() + + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + pingFail(start, "no .beads directory found") + return + } + resolveMs := time.Since(start).Milliseconds() + + st := getStore() + if st == nil { + pingFail(start, "store not initialized") + return + } + if lm, ok := storage.UnwrapStore(st).(storage.LifecycleManager); ok && lm.IsClosed() { + pingFail(start, "store is closed") + return + } + storeMs := time.Since(start).Milliseconds() + + filter := types.IssueFilter{Limit: 1} + _, err := st.SearchIssues(rootCtx, "", filter) + if err != nil { + pingFail(start, fmt.Sprintf("query failed: %v", err)) + return + } + totalMs := time.Since(start).Milliseconds() + queryMs := totalMs - storeMs + + if jsonOutput { + outputJSON(map[string]interface{}{ + "status": "ok", + "resolve_ms": resolveMs, + "store_ms": storeMs - resolveMs, + "query_ms": queryMs, + "total_ms": totalMs, + }) + return + } + + fmt.Fprintf(os.Stdout, "%s bd ping: ok (%dms)\n", ui.RenderPass("✓"), totalMs) + }, +} + +func pingFail(start time.Time, reason string) { + totalMs := time.Since(start).Milliseconds() + if jsonOutput { + outputJSON(map[string]interface{}{ + "status": "error", + "error": reason, + "total_ms": totalMs, + }) + os.Exit(1) + return + } + fmt.Fprintf(os.Stderr, "%s bd ping: %s (%dms)\n", ui.RenderFail("✗"), reason, totalMs) + os.Exit(1) +} + +func init() { + rootCmd.AddCommand(pingCmd) +} diff --git a/cmd/bd/protocol/helpers_test.go b/cmd/bd/protocol/helpers_test.go index df8ca909a..0f703f2da 100644 --- a/cmd/bd/protocol/helpers_test.go +++ b/cmd/bd/protocol/helpers_test.go @@ -505,11 +505,32 @@ func assertFieldPrefix(t *testing.T, issue map[string]any, key, prefix string) { } } -// parseJSONOutput handles both JSON array and JSONL formats. +// parseJSONOutput handles schema-versioned envelopes, JSON arrays, and JSONL. +// +// Schema-versioned output wraps arrays as {"schema_version": N, "items": [...]}. +// Object output injects schema_version as a top-level field. func parseJSONOutput(t *testing.T, output string) []map[string]any { t.Helper() - // Try JSON array first + // Try schema-versioned envelope first + var envelope map[string]any + if err := json.Unmarshal([]byte(output), &envelope); err == nil { + if items, ok := envelope["items"]; ok { + if arr, ok := items.([]any); ok { + var result []map[string]any + for _, item := range arr { + if m, ok := item.(map[string]any); ok { + result = append(result, m) + } + } + return result + } + } + // Single object (e.g. bd show --json, bd create --json) + return []map[string]any{envelope} + } + + // Try JSON array (legacy, pre-schema_version) var arr []map[string]any if err := json.Unmarshal([]byte(output), &arr); err == nil { return arr @@ -528,3 +549,16 @@ func parseJSONOutput(t *testing.T, output string) []map[string]any { } return arr } + +// assertSchemaVersion verifies that the output object contains schema_version. +func assertSchemaVersion(t *testing.T, obj map[string]any, context string) { + t.Helper() + sv, ok := obj["schema_version"] + if !ok { + t.Errorf("%s: missing schema_version field", context) + return + } + if v, ok := sv.(float64); !ok || v < 1 { + t.Errorf("%s: schema_version = %v, want >= 1", context, sv) + } +} diff --git a/cmd/bd/protocol/json_contract_test.go b/cmd/bd/protocol/json_contract_test.go index 2a1788942..471791e7d 100644 --- a/cmd/bd/protocol/json_contract_test.go +++ b/cmd/bd/protocol/json_contract_test.go @@ -19,12 +19,9 @@ func TestJSONContract_ListOutputIsValidJSON(t *testing.T) { w.create("JSON contract test issue") out := w.run("list", "--json") - var items []map[string]any - if err := json.Unmarshal([]byte(out), &items); err != nil { - t.Fatalf("bd list --json produced invalid JSON: %v\nOutput:\n%s", err, out) - } + items := parseJSONOutput(t, out) if len(items) == 0 { - t.Fatal("bd list --json returned empty array") + t.Fatal("bd list --json returned no items") } } @@ -42,7 +39,7 @@ func TestJSONContract_ShowOutputHasRequiredFields(t *testing.T) { } issue := items[0] - requiredFields := []string{"id", "title", "status", "priority", "issue_type", "created_at"} + requiredFields := []string{"id", "title", "status", "priority", "issue_type", "created_at", "schema_version"} for _, field := range requiredFields { if _, ok := issue[field]; !ok { t.Errorf("bd show --json missing required field %q", field) @@ -57,8 +54,8 @@ func TestJSONContract_ReadyOutputIsValidJSON(t *testing.T) { w := newWorkspace(t) out := w.run("ready", "--json") - var items []map[string]any - if err := json.Unmarshal([]byte(out), &items); err != nil { + var arr []map[string]any + if err := json.Unmarshal([]byte(out), &arr); err != nil { t.Fatalf("bd ready --json produced invalid JSON: %v\nOutput:\n%s", err, out) } } @@ -71,19 +68,19 @@ func TestJSONContract_CreateOutputHasID(t *testing.T) { out := w.run("create", "Create contract test", "--description=test", "--json") - // bd create --json outputs a single JSON object (not an array) var issue map[string]any if err := json.Unmarshal([]byte(out), &issue); err != nil { t.Fatalf("bd create --json produced invalid JSON: %v\nOutput:\n%s", err, out) } + assertSchemaVersion(t, issue, "bd create --json") if _, ok := issue["id"]; !ok { t.Error("bd create --json output missing 'id' field") } } // TestJSONContract_ErrorOutputIsValidJSON verifies that errors with --json -// produce valid JSON to stderr (not mixed text). +// produce valid JSON with schema_version to stderr (not mixed text). func TestJSONContract_ErrorOutputIsValidJSON(t *testing.T) { t.Parallel() w := newWorkspace(t) @@ -101,19 +98,23 @@ func TestJSONContract_ErrorOutputIsValidJSON(t *testing.T) { var errObj map[string]any if err := json.Unmarshal([]byte(trimmed), &errObj); err != nil { // Try each line — error JSON may be mixed with other stderr output - foundJSON := false for _, line := range strings.Split(trimmed, "\n") { line = strings.TrimSpace(line) if line == "" { continue } - if json.Valid([]byte(line)) { - foundJSON = true - break + var lineObj map[string]any + if json.Unmarshal([]byte(line), &lineObj) == nil { + if _, hasError := lineObj["error"]; hasError { + assertSchemaVersion(t, lineObj, "bd error JSON line") + return + } } } - if !foundJSON { - t.Logf("Note: error output not fully JSON — this is acceptable for some error paths") + t.Logf("Note: error output not fully JSON — this is acceptable for some error paths") + } else { + if _, hasError := errObj["error"]; hasError { + assertSchemaVersion(t, errObj, "bd show error --json") } } } @@ -133,3 +134,105 @@ func TestJSONContract_CloseOutputHasStatus(t *testing.T) { assertField(t, items[0], "status", "closed") } + +// TestJSONContract_ReadyOutputHasFullObjects verifies bd ready --json returns +// full issue objects with dependency counts, not just IDs (beads-clt). +func TestJSONContract_ReadyOutputHasFullObjects(t *testing.T) { + t.Parallel() + w := newWorkspace(t) + w.create("Ready full object test") + + out := w.run("ready", "--json") + items := parseJSONOutput(t, out) + if len(items) == 0 { + t.Skip("no ready issues — create returned non-ready issue") + } + issue := items[0] + requiredFields := []string{"id", "title", "status", "priority", "dependency_count", "dependent_count"} + for _, field := range requiredFields { + if _, ok := issue[field]; !ok { + t.Errorf("bd ready --json item missing required field %q", field) + } + } +} + +// TestJSONContract_BlockedOutputHasBlockedBy verifies bd blocked --json returns +// full issue objects with blocked_by field (beads-clt). +func TestJSONContract_BlockedOutputHasBlockedBy(t *testing.T) { + t.Parallel() + w := newWorkspace(t) + + blocker := w.create("Blocker issue") + blocked := w.create("Blocked issue") + w.run("dep", "add", blocked, blocker, "--type", "blocks") + + out := w.run("blocked", "--json") + items := parseJSONOutput(t, out) + + var found map[string]any + for _, item := range items { + if id, ok := item["id"].(string); ok && id == blocked { + found = item + break + } + } + if found == nil { + t.Fatalf("blocked issue %s not found in bd blocked --json output", blocked) + } + + requiredFields := []string{"id", "title", "status", "blocked_by_count", "blocked_by"} + for _, field := range requiredFields { + if _, ok := found[field]; !ok { + t.Errorf("bd blocked --json item missing required field %q", field) + } + } +} + +// TestJSONContract_PingOutputIsValidJSON verifies bd ping --json returns +// structured health check output with timing info. +func TestJSONContract_PingOutputIsValidJSON(t *testing.T) { + t.Parallel() + w := newWorkspace(t) + + out := w.run("ping", "--json") + var obj map[string]any + if err := json.Unmarshal([]byte(out), &obj); err != nil { + t.Fatalf("bd ping --json produced invalid JSON: %v\nOutput:\n%s", err, out) + } + assertSchemaVersion(t, obj, "bd ping --json") + if status, ok := obj["status"].(string); !ok || status != "ok" { + t.Errorf("bd ping --json status = %v, want ok", obj["status"]) + } + if _, ok := obj["total_ms"]; !ok { + t.Error("bd ping --json missing total_ms field") + } +} + +// TestJSONContract_SchemaVersionPresent verifies that schema_version is +// present in object-returning --json commands (show, create, ping). +// Array-returning commands (list, ready) do not include schema_version. +func TestJSONContract_SchemaVersionPresent(t *testing.T) { + t.Parallel() + w := newWorkspace(t) + id := w.create("Schema version test") + + tests := []struct { + name string + args []string + }{ + {"show", []string{"show", id, "--json"}}, + {"ping", []string{"ping", "--json"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := w.run(tt.args...) + var obj map[string]any + if err := json.Unmarshal([]byte(out), &obj); err != nil { + t.Fatalf("bd %s produced invalid JSON: %v\nOutput:\n%s", + tt.name, err, out) + } + assertSchemaVersion(t, obj, "bd "+tt.name+" --json") + }) + } +} diff --git a/cmd/bd/show_test.go b/cmd/bd/show_test.go index 96df58fb4..bfb366eb9 100644 --- a/cmd/bd/show_test.go +++ b/cmd/bd/show_test.go @@ -143,11 +143,11 @@ func TestShow_NotFoundJSON(t *testing.T) { if stdout == "" { t.Fatal("expected JSON error on stdout, got empty output") } - var errResp map[string]string + var errResp map[string]interface{} if jsonErr := json.Unmarshal([]byte(stdout), &errResp); jsonErr != nil { t.Fatalf("expected valid JSON error response on stdout, got parse error: %v\nStdout: %s", jsonErr, stdout) } - if errResp["error"] == "" { + if errField, _ := errResp["error"].(string); errField == "" { t.Errorf("expected non-empty 'error' field in JSON response, got: %s", stdout) } } diff --git a/cmd/bd/where_embedded_test.go b/cmd/bd/where_embedded_test.go index 732cbe707..c4379cb25 100644 --- a/cmd/bd/where_embedded_test.go +++ b/cmd/bd/where_embedded_test.go @@ -67,21 +67,22 @@ func TestWhereNoWorkspace(t *testing.T) { t.Fatalf("expected JSON object in output, got: %s", out) } - var payload map[string]string + var payload map[string]interface{} if err := json.Unmarshal([]byte(s[start:]), &payload); err != nil { t.Fatalf("parse where JSON: %v\n%s", err, s) } - if payload["error"] != "no_beads_directory" { - t.Fatalf("error = %q, want %q", payload["error"], "no_beads_directory") + if errField, _ := payload["error"].(string); errField != "no_beads_directory" { + t.Fatalf("error = %q, want %q", errField, "no_beads_directory") } - if payload["message"] != activeWorkspaceNotFoundMessage() { - t.Fatalf("message = %q, want %q", payload["message"], activeWorkspaceNotFoundMessage()) + if message, _ := payload["message"].(string); message != activeWorkspaceNotFoundMessage() { + t.Fatalf("message = %q, want %q", message, activeWorkspaceNotFoundMessage()) } - if !strings.Contains(payload["hint"], "BEADS_DIR/worktree setup") { - t.Fatalf("hint should mention workspace diagnostics, got: %q", payload["hint"]) + hint, _ := payload["hint"].(string) + if !strings.Contains(hint, "BEADS_DIR/worktree setup") { + t.Fatalf("hint should mention workspace diagnostics, got: %q", hint) } - if !strings.Contains(payload["hint"], "bd init") { - t.Fatalf("hint should mention bd init, got: %q", payload["hint"]) + if !strings.Contains(hint, "bd init") { + t.Fatalf("hint should mention bd init, got: %q", hint) } }) } diff --git a/docs/JSON_SCHEMA.md b/docs/JSON_SCHEMA.md new file mode 100644 index 000000000..bdeeed688 --- /dev/null +++ b/docs/JSON_SCHEMA.md @@ -0,0 +1,130 @@ +# JSON Output Schema Contract + +Commands that return a single object (show, create, close, ping, etc.) include a +`schema_version` field at the top level. Commands that return arrays (list, ready, +blocked, etc.) output a raw JSON array for backwards compatibility. + +## Schema Version + +Current version: **1** + +The `schema_version` field is an integer that increments when: +- Fields are added, renamed, or removed +- Output structure changes (e.g., nesting depth) +- Field types change (e.g., string to integer) + +Additive changes (new optional fields) do NOT bump the version. + +## Output Formats + +### Object commands (show, create, close, update, etc.) + +Commands that return a single issue or result emit a JSON object with +`schema_version` as a top-level field alongside the data: + +```json +{ + "schema_version": 1, + "id": "beads-abc", + "title": "Example issue", + "status": "open", + "priority": 1, + "issue_type": "task", + "created_at": "2026-04-20T12:00:00Z" +} +``` + +### List commands (list, ready, search, stale, etc.) + +Commands that return multiple items emit a raw JSON array: + +```json +[ + {"id": "beads-abc", "title": "First", ...}, + {"id": "beads-def", "title": "Second", ...} +] +``` + +### Error output (stderr) + +Errors with `--json` active emit JSON to stderr: + +```json +{ + "schema_version": 1, + "error": "issue not found: beads-xyz", + "code": "not_found" +} +``` + +## Field Contracts by Command + +### bd list --json + +Required fields per item: +- `id` (string): Issue ID (e.g., "beads-abc") +- `title` (string): Issue title +- `status` (string): open, in_progress, closed, deferred +- `priority` (number): 0-4 +- `issue_type` (string): bug, feature, task, epic, chore +- `created_at` (string): RFC3339 timestamp + +Optional fields: +- `description`, `owner`, `updated_at`, `closed_at` +- `labels` (string[]): Attached labels +- `dependencies` (object[]): Dependency records +- `dependency_count`, `dependent_count`, `comment_count` (number) +- `parent` (string|null): Parent issue ID + +### bd ready --json + +Same schema as `bd list --json`. Items are filtered to unblocked issues only. +Each item includes `dependency_count`, `dependent_count`, `comment_count`, +and optional `parent` fields. + +### bd blocked --json + +Returns issues that are blocked by unresolved dependencies. +Each item includes all standard issue fields plus: +- `blocked_by_count` (number): Number of blocking dependencies +- `blocked_by` (string[]): IDs of blocking issues + +### bd show --json + +Returns a single object (not wrapped in `items`). Same required fields as list +items, plus: +- `description` (string) +- `acceptance_criteria` (string) +- `dependencies` (object[]): Full dependency records +- `comments` (object[]): Comment thread + +### `import --json` + +Returns a summary object when `--json` is active: +- `source` (string): File path or "stdin" +- `created` (number): Issues created +- `skipped` (number): Issues skipped (dedup) +- `dedup_skipped` (number): Issues skipped by `--dedup` title match +- `memories` (number): Memory records imported +- `ids` (string[]): IDs of created issues +- `dry_run` (boolean): Whether `--dry-run` was active + +### bd export --json + +Outputs JSONL (one JSON object per line), not wrapped in an envelope. +Each line is a self-contained issue or memory record. `schema_version` +is included per line. + +## Consumer Guidelines + +1. **Check `schema_version`** on object output. If the version is + higher than expected, log a warning but attempt to parse anyway + (additive changes are backward-compatible). + +2. **For list commands**, parse the output as a JSON array directly. + +3. **Ignore unknown fields**. New fields may be added without bumping + the schema version. + +4. **Use `--json` flag**, not `--format json`. The `--json` flag is + the stable contract; `--format` is for human-readable variants.