Add configurable JSONL import path

This commit is contained in:
Medhat Galal 2026-05-19 13:37:32 -04:00
commit 507b2a1fb5
19 changed files with 162 additions and 24 deletions

View file

@ -24,8 +24,8 @@ type jsonlImporter interface {
// non-empty database. Production builds always use importFromLocalJSONLFull.
var fallbackImporter = importFromLocalJSONLFull
// maybeAutoImportJSONL checks whether the database is empty and a
// issues.jsonl file exists in beadsDir. When both conditions are true it
// maybeAutoImportJSONL checks whether the database is empty and the configured
// import.path JSONL file exists in beadsDir. When both conditions are true it
// auto-imports the JSONL data so users upgrading from pre-0.56 (which used
// .beads/dolt/) to 1.0+ (which uses .beads/embeddeddolt/) don't appear to
// lose their issues. See GH#2994.
@ -43,7 +43,7 @@ var fallbackImporter = importFromLocalJSONLFull
// prevent the store from being used.
func maybeAutoImportJSONL(ctx context.Context, s storage.DoltStorage, beadsDir string) {
// Quick check: does the JSONL file exist and have content?
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
jsonlPath := configuredImportJSONLPath(beadsDir)
info, err := os.Stat(jsonlPath)
if err != nil || info.Size() == 0 {
return // no JSONL file or empty — nothing to import

View file

@ -8,6 +8,7 @@ import (
"sync/atomic"
"testing"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
)
@ -31,12 +32,17 @@ func (f *fakeFallbackStore) GetStatistics(_ context.Context) (*types.Statistics,
}
func writeAutoImportFixtureJSONL(t *testing.T, dir string) {
t.Helper()
writeAutoImportFixtureJSONLNamed(t, dir, "issues.jsonl")
}
func writeAutoImportFixtureJSONLNamed(t *testing.T, dir, name string) {
t.Helper()
// Minimal valid issue line. Contents are irrelevant for the
// skip-when-non-empty test (returns before parseJSONLFile) and are
// only required to be parseable for the negative-control test.
line := `{"_type":"issue","id":"unit-1","title":"unit-test-fixture","status":"open","priority":2,"issue_type":"task"}`
if err := os.WriteFile(filepath.Join(dir, "issues.jsonl"), []byte(line+"\n"), 0o600); err != nil {
if err := os.WriteFile(filepath.Join(dir, name), []byte(line+"\n"), 0o600); err != nil {
t.Fatal(err)
}
}
@ -117,3 +123,27 @@ func TestMaybeAutoImportJSONL_ServerModeFallback_RunsWhenEmpty(t *testing.T) {
t.Fatalf("server-mode fallback importer invoked %d time(s) on empty store; expected exactly 1", got)
}
}
func TestMaybeAutoImportJSONL_UsesConfiguredImportPath(t *testing.T) {
initConfigForTest(t)
config.Set("import.path", "beads.jsonl")
dir := t.TempDir()
writeAutoImportFixtureJSONLNamed(t, dir, "beads.jsonl")
orig := fallbackImporter
var gotPath string
fallbackImporter = func(_ context.Context, _ storage.DoltStorage, path string) (*importLocalResult, error) {
gotPath = path
return nil, errors.New("test importer: short-circuit before s.Commit")
}
t.Cleanup(func() { fallbackImporter = orig })
store := &fakeFallbackStore{statsTotalIssues: 0}
maybeAutoImportJSONL(context.Background(), store, dir)
wantPath := filepath.Join(dir, "beads.jsonl")
if gotPath != wantPath {
t.Fatalf("auto-import path = %q, want %q", gotPath, wantPath)
}
}

View file

@ -28,6 +28,7 @@ Configuration is stored per-project in the beads database and is version-control
Common namespaces:
- export.* Auto-export settings (stored in config.yaml)
- import.* JSONL import settings (stored in config.yaml)
- jira.* Jira integration settings
- linear.* Linear integration settings
- github.* GitHub integration settings
@ -46,6 +47,12 @@ Auto-Export (config.yaml):
export.interval Minimum time between exports (default: 60s)
export.git-add Auto-stage the export file (default: true)
Auto-Import (config.yaml):
Reads .beads/issues.jsonl by default when a JSONL import path is implied.
Keys:
import.path Input filename relative to .beads/ (default: issues.jsonl)
Custom Status States:
You can define custom status states for multi-step pipelines using the
status.custom config key. Statuses should be comma-separated.
@ -67,6 +74,7 @@ Suppressing Doctor Warnings:
Examples:
bd config set export.auto false # Disable auto-export
bd config set export.path "beads.jsonl" # Custom export filename
bd config set import.path "beads.jsonl" # Custom import filename
bd config set jira.url "https://company.atlassian.net"
bd config set jira.project "PROJ"
bd config set status.custom "awaiting_review,awaiting_testing"
@ -758,7 +766,7 @@ Examples:
// recognizedConfigPrefixes lists valid top-level config namespaces.
// Keys under custom.* are always accepted (user-extensible).
var recognizedConfigPrefixes = []string{
"export.", "dolt.", "jira.", "linear.", "github.", "custom.",
"export.", "import.", "dolt.", "jira.", "linear.", "github.", "custom.",
"status.", "doctor.suppress.", "routing.", "sync.", "git.",
"directory.", "repos.", "external_projects.", "validation.",
"hierarchy.", "ai.", "backup.", "federation.",

View file

@ -53,7 +53,7 @@ func TestConfigSetManyArgParsing(t *testing.T) {
// TestConfigSetManyYamlKeyDetection tests that yaml-only keys are correctly identified
// for routing in the set-many command.
func TestConfigSetManyYamlKeyDetection(t *testing.T) {
yamlKeys := []string{"no-db", "json", "routing.mode", "routing.default", "no-push"}
yamlKeys := []string{"no-db", "json", "routing.mode", "routing.default", "no-push", "import.path"}
for _, key := range yamlKeys {
if !config.IsYamlOnlyKey(key) {
t.Errorf("expected %q to be yaml-only", key)

View file

@ -9,7 +9,7 @@ func TestIsRecognizedConfigKey(t *testing.T) {
recognized := []string{
"export.auto", "dolt.auto-push", "jira.url", "custom.anything",
"doctor.suppress.git-hooks", "no-git-ops", "beads.role",
"status.custom", "ai.model", "backup.enabled",
"status.custom", "ai.model", "backup.enabled", "import.path",
}
for _, key := range recognized {
if !isRecognizedConfigKey(key) {

View file

@ -1388,7 +1388,7 @@ func exportSubprocessDir(beadsDir string) string {
return filepath.Dir(beadsDir)
}
// importJSONLForSync imports .beads/issues.jsonl into Dolt after a git
// importJSONLForSync imports the configured import.path JSONL into Dolt after a git
// pull/merge/branch-checkout only for legacy projects with no Dolt remote.
// When sync.remote is configured, Dolt remains the source of truth and JSONL
// import is skipped because upsert-only import cannot reconcile stale exports.
@ -1412,11 +1412,7 @@ func importJSONLForSync(reason string) {
return
}
exportPath := config.GetString("export.path")
if exportPath == "" {
exportPath = "issues.jsonl"
}
fullPath := filepath.Join(beadsDir, exportPath)
fullPath := configuredImportJSONLPath(beadsDir)
if info, err := os.Stat(fullPath); err != nil || info.Size() == 0 {
return

View file

@ -21,8 +21,8 @@ var importCmd = &cobra.Command{
Short: "Import issues from a JSONL file or stdin into the database",
Long: `Import issues from a JSONL file (newline-delimited JSON) into the database.
If no file is specified, imports from .beads/issues.jsonl (the git-tracked
export). Use "-" to read from stdin. This is the incremental counterpart to
If no file is specified, imports from the configured import.path under .beads/
(default: issues.jsonl). Use "-" to read from stdin. This is the incremental counterpart to
'bd export': new issues are created and existing issues are updated (upsert
semantics).
@ -58,7 +58,7 @@ when present in the JSONL and otherwise filled in by the importer. The
legacy "wisp" boolean is accepted as an alias for "ephemeral".
EXAMPLES:
bd import # Import from .beads/issues.jsonl
bd import # Import from configured import.path
bd import backup.jsonl # Import from a specific file
bd import -i backup.jsonl # Legacy alias for a specific file
bd import - # Read JSONL from stdin
@ -109,7 +109,7 @@ func runImport(cmd *cobra.Command, args []string) error {
if globalFlag {
jsonlPath = filepath.Join(beadsDir, "global-issues.jsonl")
} else {
jsonlPath = filepath.Join(beadsDir, "issues.jsonl")
jsonlPath = configuredImportJSONLPath(beadsDir)
}
}

View file

@ -89,6 +89,28 @@ func TestEmbeddedImport(t *testing.T) {
}
})
t.Run("from_configured_import_path", func(t *testing.T) {
dir, _, _ := bdInit(t, bd, "--prefix", "imcfg")
cmd := exec.Command(bd, "config", "set", "import.path", "beads.jsonl")
cmd.Dir = dir
cmd.Env = bdEnv(dir)
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("bd config set import.path failed: %v\n%s", err, out)
}
jsonlPath := filepath.Join(dir, ".beads", "beads.jsonl")
now := time.Now().UTC()
writeJSONLFile(t, jsonlPath, []types.Issue{
{ID: "imcfg-xxx", Title: "Configured Import Path Issue", Status: types.StatusOpen, IssueType: types.TypeTask, CreatedAt: now, UpdatedAt: now},
})
out := bdImport(t, bd, dir)
if !strings.Contains(out, "Imported 1 issue") {
t.Errorf("expected 'Imported 1 issue', got: %s", out)
}
})
t.Run("dry_run", func(t *testing.T) {
dir, _, _ := bdInit(t, bd, "--prefix", "imdry")

17
cmd/bd/import_path.go Normal file
View file

@ -0,0 +1,17 @@
package main
import (
"path/filepath"
"github.com/steveyegge/beads/internal/config"
)
const defaultImportJSONLPath = "issues.jsonl"
func configuredImportJSONLPath(beadsDir string) string {
importPath := config.GetString("import.path")
if importPath == "" {
importPath = defaultImportJSONLPath
}
return filepath.Join(beadsDir, importPath)
}

View file

@ -1064,7 +1064,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1):
// Import from local JSONL if requested (GH#2023).
// This must run after the store is created and prefix is set.
if fromJSONL {
localJSONLPath := filepath.Join(beadsDir, "issues.jsonl")
localJSONLPath := configuredImportJSONLPath(beadsDir)
if _, statErr := os.Stat(localJSONLPath); os.IsNotExist(statErr) {
_ = store.Close()
FatalError("--from-jsonl specified but %s does not exist", localJSONLPath)
@ -1484,7 +1484,7 @@ func init() {
initCmd.Flags().Bool("force", false, "Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence (see 'bd help init-safety').")
initCmd.Flags().Bool("reinit-local", false, "Re-initialize local .beads/ over existing local data. Does NOT authorize remote divergence; see --discard-remote.")
initCmd.Flags().Bool("discard-remote", false, "Authorize discarding the configured remote's Dolt history when re-initializing. Requires --destroy-token in non-interactive mode; see 'bd help init-safety'.")
initCmd.Flags().Bool("from-jsonl", false, "Import issues from .beads/issues.jsonl instead of git history")
initCmd.Flags().Bool("from-jsonl", false, "Import issues from configured import.path instead of git history")
initCmd.Flags().String("destroy-token", "", "Explicit confirmation token for destructive re-init in non-interactive mode (format: 'DESTROY-<prefix>')")
initCmd.Flags().String("agents-template", "", "Path to custom AGENTS.md template (overrides embedded default)")
initCmd.Flags().String("agents-profile", "", "AGENTS.md profile: 'minimal' (default, pointer to bd prime) or 'full' (complete command reference)")

View file

@ -876,6 +876,46 @@ func TestEmbeddedInit(t *testing.T) {
}
})
t.Run("from_jsonl_uses_import_path", func(t *testing.T) {
dir := t.TempDir()
initGitRepoAt(t, dir)
beadsDir := filepath.Join(dir, ".beads")
if err := os.MkdirAll(beadsDir, 0750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("import:\n path: beads.jsonl\n"), 0600); err != nil {
t.Fatal(err)
}
issue := types.Issue{
ID: "jlcfg-abc123",
Title: "Configured JSONL",
Status: types.StatusOpen,
Priority: 2,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
line, _ := json.Marshal(issue)
if err := os.WriteFile(filepath.Join(beadsDir, "beads.jsonl"), append(line, '\n'), 0644); err != nil {
t.Fatal(err)
}
cmd := exec.Command(bd, "init", "--prefix", "jlcfg", "--from-jsonl", "--quiet")
cmd.Dir = dir
cmd.Env = bdEnv(dir)
stdout, stderr, err := runCommandBuffers(t, cmd)
if err != nil {
t.Fatalf("--from-jsonl with import.path failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String())
}
showCmd := exec.Command(bd, "show", "jlcfg-abc123", "--json")
showCmd.Dir = dir
showCmd.Env = bdEnv(dir)
if out, err := showCmd.CombinedOutput(); err != nil {
t.Fatalf("imported issue not found: %v\n%s", err, out)
}
})
t.Run("backend_dolt", func(t *testing.T) {
_, beadsDir, _ := bdInit(t, bd, "--prefix", "bdolt", "--backend", "dolt")
embeddedDir := filepath.Join(beadsDir, "embeddeddolt")

View file

@ -73,6 +73,12 @@ func createConfigYaml(beadsDir string, noDbMode bool, prefix string) error {
# git-push: false # Disable git push (export locally only)
# git-repo: "" # Separate git repo for backups (default: project repo)
# JSONL import/export filenames (relative to .beads/)
# export:
# path: issues.jsonl
# import:
# path: issues.jsonl
# Integration settings (access with 'bd config get/set')
# Non-secret keys (stored in the database):
# - jira.url, jira.project

View file

@ -28,7 +28,8 @@ func validateKVKey(key string) error {
// Prevent keys that look like internal config
if strings.HasPrefix(key, "sync.") || strings.HasPrefix(key, "conflict.") ||
strings.HasPrefix(key, "federation.") || strings.HasPrefix(key, "jira.") ||
strings.HasPrefix(key, "linear.") || strings.HasPrefix(key, "export.") {
strings.HasPrefix(key, "linear.") || strings.HasPrefix(key, "export.") ||
strings.HasPrefix(key, "import.") {
return fmt.Errorf("key cannot start with reserved prefix %q", strings.Split(key, ".")[0]+".")
}
return nil

View file

@ -202,6 +202,7 @@ func TestValidateKVKey(t *testing.T) {
{"jira prefix", "jira.url", true, "reserved prefix"},
{"linear prefix", "linear.key", true, "reserved prefix"},
{"export prefix", "export.path", true, "reserved prefix"},
{"import prefix", "import.path", true, "reserved prefix"},
}
for _, tc := range testCases {

View file

@ -2370,8 +2370,8 @@ bd federation
Import issues from a JSONL file (newline-delimited JSON) into the database.
If no file is specified, imports from .beads/issues.jsonl (the git-tracked
export). Use "-" to read from stdin. This is the incremental counterpart to
If no file is specified, imports from the configured import.path under .beads/
(default: issues.jsonl). Use "-" to read from stdin. This is the incremental counterpart to
'bd export': new issues are created and existing issues are updated (upsert
semantics).
@ -2407,7 +2407,7 @@ when present in the JSONL and otherwise filled in by the importer. The
legacy "wisp" boolean is accepted as an alias for "ephemeral".
EXAMPLES:
bd import # Import from .beads/issues.jsonl
bd import # Import from configured import.path
bd import backup.jsonl # Import from a specific file
bd import -i backup.jsonl # Legacy alias for a specific file
bd import - # Read JSONL from stdin
@ -2566,6 +2566,7 @@ Configuration is stored per-project in the beads database and is version-control
Common namespaces:
- export.* Auto-export settings (stored in config.yaml)
- import.* JSONL import settings (stored in config.yaml)
- jira.* Jira integration settings
- linear.* Linear integration settings
- github.* GitHub integration settings
@ -2584,6 +2585,12 @@ Auto-Export (config.yaml):
export.interval Minimum time between exports (default: 60s)
export.git-add Auto-stage the export file (default: true)
Auto-Import (config.yaml):
Reads .beads/issues.jsonl by default when a JSONL import path is implied.
Keys:
import.path Input filename relative to .beads/ (default: issues.jsonl)
Custom Status States:
You can define custom status states for multi-step pipelines using the
status.custom config key. Statuses should be comma-separated.
@ -2605,6 +2612,7 @@ Suppressing Doctor Warnings:
Examples:
bd config set export.auto false # Disable auto-export
bd config set export.path "beads.jsonl" # Custom export filename
bd config set import.path "beads.jsonl" # Custom import filename
bd config set jira.url "https://company.atlassian.net"
bd config set jira.project "PROJ"
bd config set status.custom "awaiting_review,awaiting_testing"
@ -3363,7 +3371,7 @@ bd init [flags]
--discard-remote Authorize discarding the configured remote's Dolt history when re-initializing. Requires --destroy-token in non-interactive mode; see 'bd help init-safety'.
--external Server is externally managed (skip server startup); use with --shared-server or --server
--force Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence (see 'bd help init-safety').
--from-jsonl Import issues from .beads/issues.jsonl instead of git history
--from-jsonl Import issues from configured import.path instead of git history
--non-interactive Skip all interactive prompts (auto-detected in CI or non-TTY environments)
-p, --prefix string Issue prefix (default: current directory name)
--proxied-server [EXPERIMENTAL] Use a per-workspace proxied dolt sql-server (proxy + child dolt) rooted at .beads/proxieddb

View file

@ -342,6 +342,7 @@ Configuration keys use dot-notation namespaces to organize settings:
- `min_hash_length` - Minimum hash ID length (default: 4)
- `max_hash_length` - Maximum hash ID length (default: 8)
- `import.orphan_handling` - How to handle hierarchical issues with missing parents during import (default: `allow`)
- `import.path` - Input filename relative to `.beads/` for implied JSONL imports, including `bd init --from-jsonl` and empty-DB auto-import (default: `issues.jsonl`)
- `export.auto` - Refresh the JSONL export after every write command (default: `true`). This is for viewers, interchange, and backup, not cross-machine sync.
- `export.path` - Output filename relative to `.beads/` (default: `issues.jsonl`)
- `export.interval` - Minimum time between auto-exports (default: `60s`)

View file

@ -249,6 +249,7 @@ func Initialize() error {
// configured a Dolt remote yet. Hook code skips this path when sync.remote
// is configured because JSONL import is upsert-only, not reconciliation.
v.SetDefault("import.auto", true)
v.SetDefault("import.path", "issues.jsonl") // relative to .beads/; canonical import name
// AI configuration defaults
v.SetDefault("ai.model", "claude-haiku-4-5-20251001")

View file

@ -63,6 +63,9 @@ var YamlOnlyKeys = map[string]bool{
"backup.git-push": true,
"backup.git-repo": true,
// Import settings
"import.path": true,
// Dolt server settings
"dolt.shared-server": true, // Shared Dolt server at ~/.beads/shared-server/ (GH#2377)
"dolt.max-conns": true, // Connection pool size override (default 10, GH#3140)

View file

@ -40,6 +40,10 @@ func TestIsYamlOnlyKey(t *testing.T) {
{"backup.git-repo", true},
{"backup.future-key", true}, // prefix match
// Import settings
{"import.path", true},
{"import.orphan_handling", false},
// Secret keys (stored in yaml to avoid leaking via Dolt push)
{"github.token", true},
{"linear.api_key", true},