diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 2e3f8856b..144e4310e 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "database/sql" + "encoding/json" "errors" "fmt" "io" @@ -22,6 +23,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/storage/embeddeddolt" @@ -51,6 +53,12 @@ type bootstrapServerDBCheck struct { // retries. Injectable for testing. var bootstrapRetryDelay = func(d time.Duration) { time.Sleep(d) } +// probeGitRemoteDoltData reports whether a git remote carries Dolt data +// (refs/dolt/data). Injectable for testing; the production value is a +// 10s-bounded `git ls-remote` with credential prompts suppressed. A non-nil +// error means UNKNOWN, not "no data". +var probeGitRemoteDoltData = gitRemoteHasDoltDataRefStatus + var checkBootstrapServerDB = func(probeCfg bootstrapServerProbeConfig) bootstrapServerDBCheck { host := probeCfg.host port := probeCfg.port @@ -113,6 +121,9 @@ Bootstrap auto-detects the right action: • If no database exists: creates a fresh one • If database already exists: validates and reports status +If sync.remote points at a git repository, bootstrap verifies refs/dolt/data +before cloning. Bootstrap exits non-zero when it cannot set up a database. + This is the recommended command for: • Setting up beads on a fresh clone • Recovering after moving to a new machine @@ -231,7 +242,7 @@ Examples: return err } if plan.Action == "none" || dryRun { - return nil + return bootstrapPlanOutcome(plan, dryRun) } } else { if repairMsg != "" { @@ -239,7 +250,7 @@ Examples: } printBootstrapPlan(plan) if plan.Action == "none" || dryRun { - return nil + return bootstrapPlanOutcome(plan, dryRun) } } @@ -269,14 +280,73 @@ func applyBootstrapMetadataRepair(beadsDir string, cfg *configfile.Config, apply // BootstrapPlan describes what bootstrap will do. type BootstrapPlan struct { - Action string `json:"action"` // "sync", "restore", "jsonl-import", "init", "none" - Reason string `json:"reason"` // Human-readable explanation - BeadsDir string `json:"beads_dir"` - Database string `json:"database"` - SyncRemote string `json:"sync_remote,omitempty"` - BackupDir string `json:"backup_dir,omitempty"` - JSONLFile string `json:"jsonl_file,omitempty"` - HasExisting bool `json:"has_existing"` + Action string `json:"action"` // "sync", "restore", "jsonl-import", "init", "none" + Reason string `json:"reason"` // Human-readable explanation + BeadsDir string `json:"beads_dir"` + Database string `json:"database"` + // SyncRemote is the remote this workspace syncs with. For Action=="sync" + // it is the clone source; for the provisioning actions it is the remote to + // register as Dolt "origin" so the first `bd dolt push` has somewhere to + // go. Always the routed (doltRemoteURL) form for forge URLs, and empty on + // a Blocked plan — nothing may be cloned from or pushed to a remote + // bootstrap could not verify. + // + // This is an operational URL, so it keeps whatever credentials the user + // configured. Run it through redactRemoteURL before printing it or putting + // it anywhere a human will read. Reason and BlockedRemote are already + // display-safe. + SyncRemote string `json:"sync_remote,omitempty"` + BackupDir string `json:"backup_dir,omitempty"` + JSONLFile string `json:"jsonl_file,omitempty"` + // Blocked marks a plan that took no action and found no database. It is + // derived in detectBootstrapAction from Action+HasExisting, never set by a + // detection branch, and it is the single discriminator the exit code and + // the printed output both switch on. Additive on purpose: JSON consumers + // switch on "action", so a new action value would break them, while + // "none" + "blocked" + a non-zero exit code is backwards compatible. + Blocked bool `json:"blocked,omitempty"` + // BlockedRemote is the credential-free remote URL a Blocked plan could not + // verify, carried only so the diagnostic can name it. + BlockedRemote string `json:"blocked_remote,omitempty"` + HasExisting bool `json:"has_existing"` +} + +// MarshalJSON renders the plan for `bd bootstrap --json` (and `--dry-run --json`) +// with the credential-bearing SyncRemote redacted, mirroring the human path +// (printBootstrapPlan runs it through redactRemoteURL). SyncRemote keeps whatever +// credentials the user configured — see the field comment — and `--json` writes +// to stdout, which CI logs and agent transcripts capture, so it must not be +// emitted verbatim. Redaction is on a copy; executeBootstrapPlan still consumes +// the raw plan.SyncRemote for the clone. Reason and BlockedRemote are already +// built display-safe. +func (p BootstrapPlan) MarshalJSON() ([]byte, error) { + // The local type sheds BootstrapPlan's method set so json.Marshal does not + // recurse back into this MarshalJSON. + type bootstrapPlanJSON BootstrapPlan + safe := bootstrapPlanJSON(p) + safe.SyncRemote = redactRemoteURL(p.SyncRemote) + return json.Marshal(safe) +} + +// bootstrapPlanOutcome maps a printed plan to the command's exit status. +// Action=="none" was two outcomes wearing one name — "a database already +// exists, nothing to do" and "bootstrap declined and left you with no +// database" — and both returned nil, so a rejected sync.remote printed a +// success tick and exited 0 with no database (#5743). Blocked is the +// discriminator. +// +// --dry-run always succeeds. bd doctor documents `bd bootstrap --dry-run` as +// the safe inspection step, and a preview that aborts a `set -e` provisioning +// script is a worse contract than a preview that merely reports. The blocked +// state is still printed (and still in the JSON), so the preview does predict +// what a real run would refuse to do. +func bootstrapPlanOutcome(plan BootstrapPlan, dryRun bool) error { + if dryRun || !plan.Blocked { + return nil + } + // The reason and hints are already on stderr, or the plan JSON (carrying + // "blocked": true) is already on stdout. + return SilentExit() } func noWorkspaceBootstrapPayload() map[string]interface{} { @@ -304,7 +374,20 @@ func requireBootstrapDoltBackend(cfg *configfile.Config, beadsDir string) error return nil } +// detectBootstrapAction builds the plan and then derives its outcome flags in +// exactly one place. Blocked is never hand-set by a detection branch: it is a +// function of the finished plan ("took no action AND found no database"), so no +// branch can leave a stale flag behind for a later branch to inherit. func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPlan { + plan := detectBootstrapPlan(beadsDir, cfg) + plan.Blocked = plan.Action == "none" && !plan.HasExisting + if !plan.Blocked { + plan.BlockedRemote = "" + } + return plan +} + +func detectBootstrapPlan(beadsDir string, cfg *configfile.Config) BootstrapPlan { plan := BootstrapPlan{ BeadsDir: beadsDir, Database: cfg.GetDoltDatabase(), @@ -315,10 +398,8 @@ func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPla // sync.remote / refs/dolt/data detection run before treating an existing // shared-server database as "nothing to do", otherwise an unrelated default // "beads" database can mask the real recovery path. - beadsDirExists := false - if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { - beadsDirExists = true - } + info, statErr := os.Stat(beadsDir) + beadsDirExists := statErr == nil && info.IsDir() // Check for existing database (path differs between server and embedded mode). // Determine server/shared-server mode from the target workspace itself @@ -335,68 +416,57 @@ func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPla // and the multi-clone upgrade guide. If the local beadsDir does not exist // yet, still prefer sync recovery first for Action=="none" so a default // shared-server "beads" DB from another project cannot mask a real clone. + // Held aside rather than merged into plan: a "none" verdict here is only a + // fallback for when nothing else matches, and copying it into plan leaked + // its HasExisting/Blocked/Reason into every branch below — including the + // probe-error branch, which then inherited HasExisting=true and printed the + // #5743 success tick again. + var deferredExistingPlan *BootstrapPlan if dbAction, ok := existingBootstrapDBPlan(beadsDir, cfg, isServer, isSharedServer); ok { if beadsDirExists || dbAction.Action != "none" { return dbAction } - plan = dbAction + deferredExistingPlan = &dbAction } - // Check sync.remote (primary) or sync.git-remote (deprecated fallback) + // A configured sync.remote (primary) or sync.git-remote (deprecated + // fallback) is authoritative: try it first and, when it cannot be cloned, + // remember why so an unverifiable remote can become fatal only after local + // recovery is exhausted. With no sync.remote, auto-detect Dolt data on git + // origin instead. syncRemote := resolveSyncRemote() + unusableRemoteReason, unusableRemoteURL := "", "" if syncRemote != "" { - if isGitCodeRepoURL(syncRemote) { - // Cloning from a git code-repo URL via DOLT_CLONE spins dolt to - // 1000% CPU and requires manual SIGKILL. Reject and - // surface the misconfiguration rather than attempting the clone. - fmt.Fprintf(os.Stderr, "error: sync.remote %q looks like a git code-repository URL, not a Dolt remote — skipping clone\n", syncRemote) - plan.Action = "none" - plan.Reason = fmt.Sprintf("sync.remote %q rejected: git code-repository URL (not a Dolt remote)", syncRemote) + done, reason, url := planConfiguredSyncRemote(&plan, syncRemote) + if done { return plan } - // User-provided sync.remote — trust the URL format as-is. - // normalizeRemoteURL would convert http:// to git+http://, - // breaking Dolt remotesapi endpoints (GH#3339). - plan.SyncRemote = syncRemote - plan.Action = "sync" - plan.Reason = "sync.remote configured — will clone from " + syncRemote - return plan + // Cloning is off the table; purely local recovery is not. + unusableRemoteReason, unusableRemoteURL = reason, url + } else if originPlan, ok := gitOriginSyncPlan(plan); ok { + return originPlan } - // Auto-detect: probe git origin for Dolt data stored in git - // (refs/dolt/data). This only applies to git remotes — Dolt-native - // remotes (DoltHub, S3, etc.) must be configured via sync.remote. - if isGitRepo() && !isBareGitRepo() { - if originURL, err := gitOriginGetURL(); err == nil && originURL != "" { - if gitOriginHasDoltDataRef() { - plan.SyncRemote = normalizeRemoteURL(originURL) - plan.Action = "sync" - plan.Reason = "Found Dolt data on git origin (refs/dolt/data) — will clone from " + originURL - return plan - } - } + // On-disk recovery sources: a non-empty backup JSONL, then a git-tracked + // issues.jsonl. When a forge sync.remote had no Dolt data yet, plan already + // carries its SyncRemote so the restored/imported workspace stays wired to it. + if recoveryPlan, ok := localFileRecoveryPlan(plan, beadsDir); ok { + return recoveryPlan } - // Check for backup JSONL files (must be non-empty to be useful) - backupDir := filepath.Join(beadsDir, "backup") - issuesFile := filepath.Join(backupDir, "issues.jsonl") - if info, err := os.Stat(issuesFile); err == nil && info.Size() > 0 { - plan.BackupDir = backupDir - plan.Action = "restore" - plan.Reason = "Backup files found — will restore from " + backupDir - return plan + if deferredExistingPlan != nil { + return *deferredExistingPlan } - // Check for git-tracked JSONL (the portable export format) - gitJSONL := filepath.Join(beadsDir, "issues.jsonl") - if _, err := os.Stat(gitJSONL); err == nil { - plan.JSONLFile = gitJSONL - plan.Action = "jsonl-import" - plan.Reason = "Git-tracked issues.jsonl found — will import from " + gitJSONL - return plan - } - - if plan.Action == "none" { + // Only now, with every local recovery source exhausted, does an + // unverifiable remote become fatal. Restoring a local backup touches no + // remote, so an offline laptop or a credential-less CI runner must not be + // denied the recovery that the same machine would get if the probe had + // merely answered "no data". + if unusableRemoteReason != "" { + plan.Action = "none" + plan.Reason = unusableRemoteReason + plan.BlockedRemote = unusableRemoteURL return plan } @@ -406,6 +476,116 @@ func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPla return plan } +// planConfiguredSyncRemote resolves a configured sync.remote into plan. done==true +// means the plan is finished — a clone plan for a trusted remote or a forge repo +// the probe confirmed carries Dolt data — and the caller returns plan as-is. +// done==false means keep looking: either the forge repo has no Dolt data yet +// (plan.SyncRemote is still recorded so a fresh workspace gets wired to it) or the +// remote could not be verified, in which case the returned reason/url describe the +// unusable remote so it can become fatal only after local recovery is exhausted. +func planConfiguredSyncRemote(plan *BootstrapPlan, syncRemote string) (done bool, unusableReason, unusableURL string) { + if !isGitCodeRepoURL(syncRemote) { + // User-provided sync.remote — trust the URL format as-is. + // normalizeRemoteURL would convert http:// to git+http://, breaking Dolt + // remotesapi endpoints (GH#3339). That concern cannot apply here: a + // remotesapi endpoint never classifies as a git code-repo URL. + plan.SyncRemote = syncRemote + plan.Action = "sync" + plan.Reason = "sync.remote configured — will clone from " + redactRemoteURL(syncRemote) + return true, "", "" + } + + // A git-forge URL is a *supported* Dolt remote: `bd init` and `bd dolt + // remote add` both persist one, and `bd dolt push` stores the database under + // refs/dolt/data on it. What must never happen is cloning it blind, or in a + // form that reaches Dolt's remotesapi client. doltRemoteURL owns that routing + // rule; probe first, and clone only what the probe actually confirmed + // (#5743, #5663). + routed := doltRemoteURL(syncRemote) + hasData, probeErr := probeGitRemoteDoltData(routed) + switch { + case probeErr != nil: + // UNKNOWN, not "no data" — never clone something we could not verify + // (init's resolveRemoteHasDoltDataProbe convention). + reason := fmt.Sprintf("could not verify refs/dolt/data on sync.remote %q: %v", + redactRemoteURL(syncRemote), probeErr) + // Said here as well as in the final verdict because a local backup may + // well rescue this run, and then this is the only place the user learns + // the remote could not be reached. + fmt.Fprintf(os.Stderr, "note: not cloning — %s; checking local recovery sources\n", reason) + return false, reason, redactRemoteURL(routed) + case hasData: + plan.SyncRemote = routed + plan.Action = "sync" + plan.Reason = "sync.remote git repo has Dolt data (refs/dolt/data) — will clone from " + redactRemoteURL(routed) + return true, "", "" + default: + // Definitively no Dolt data: the repo was wired before the first `bd + // dolt push`. Say so and keep looking, the same way bd init falls back + // to a fresh local database. Carry the URL so the workspace we create + // still gets wired to it. + plan.SyncRemote = routed + fmt.Fprintf(os.Stderr, "note: sync.remote %q has no Dolt data yet (refs/dolt/data absent); checking other recovery sources\n", + redactRemoteURL(syncRemote)) + return false, "", "" + } +} + +// gitOriginSyncPlan returns a clone-from-git-origin plan when git origin carries +// Dolt data (refs/dolt/data), or ok==false otherwise. Only git remotes qualify — +// Dolt-native remotes (DoltHub, S3, etc.) must be configured via sync.remote. +// +// The caller invokes this only when no sync.remote is configured. sync.remote is +// the authoritative remote for this workspace, so hydrating from whatever git +// origin happens to be (a fork, an old experiment, the code repo under the +// dedicated-data-repo pattern) is not recovery — and finalizeSyncedBootstrap would +// then rewrite the team's committed sync.remote to point at it. Skipping it also +// spares the second identical ls-remote in the common case where sync.remote IS +// the git origin, which is what bd init persists. +func gitOriginSyncPlan(plan BootstrapPlan) (BootstrapPlan, bool) { + if !isGitRepo() || isBareGitRepo() { + return BootstrapPlan{}, false + } + originURL, err := gitOriginGetURL() + if err != nil || originURL == "" { + return BootstrapPlan{}, false + } + if !gitOriginHasDoltDataRef() { + return BootstrapPlan{}, false + } + plan.SyncRemote = normalizeRemoteURL(originURL) + plan.Action = "sync" + plan.Reason = "Found Dolt data on git origin (refs/dolt/data) — will clone from " + redactRemoteURL(originURL) + return plan, true +} + +// localFileRecoveryPlan returns a restore or jsonl-import plan built from on-disk +// recovery sources under beadsDir — a non-empty backup/issues.jsonl (restore), +// else a git-tracked issues.jsonl (the portable export format) — or ok==false +// when neither is present. plan is taken by value so its already-populated fields +// (e.g. a forge SyncRemote with no Dolt data yet) ride into the returned plan. +func localFileRecoveryPlan(plan BootstrapPlan, beadsDir string) (BootstrapPlan, bool) { + // Backup JSONL must be non-empty to be useful. + backupDir := filepath.Join(beadsDir, "backup") + issuesFile := filepath.Join(backupDir, "issues.jsonl") + if info, err := os.Stat(issuesFile); err == nil && info.Size() > 0 { + plan.BackupDir = backupDir + plan.Action = "restore" + plan.Reason = "Backup files found — will restore from " + backupDir + return plan, true + } + + gitJSONL := filepath.Join(beadsDir, "issues.jsonl") + if _, err := os.Stat(gitJSONL); err == nil { + plan.JSONLFile = gitJSONL + plan.Action = "jsonl-import" + plan.Reason = "Git-tracked issues.jsonl found — will import from " + gitJSONL + return plan, true + } + + return BootstrapPlan{}, false +} + func existingBootstrapDBPlan(beadsDir string, cfg *configfile.Config, isServer, isSharedServer bool) (BootstrapPlan, bool) { plan := BootstrapPlan{ BeadsDir: beadsDir, @@ -455,6 +635,11 @@ func existingBootstrapDBPlan(beadsDir string, cfg *configfile.Config, isServer, bootstrapRetryDelay(retryDelays[attempt]) } if result.Err != nil { + // Same "declined, and no database was confirmed" class as a + // failed remote probe: report it and exit non-zero rather than + // printing a success tick (#5743). Blocked is not hand-set here — + // detectBootstrapAction derives it from Action=="none" && !HasExisting + // (HasExisting is false on this branch), keeping one owner for the flag. plan.Action = "none" plan.Reason = fmt.Sprintf("Could not verify existing server database %s: %v", cfg.GetDoltDatabase(), result.Err) return plan, true @@ -534,6 +719,22 @@ func bootstrapServerPort(beadsDir string, cfg *configfile.Config, isSharedServer func printBootstrapPlan(plan BootstrapPlan) { switch plan.Action { case "none": + if plan.Blocked { + // Nothing was set up and nothing was found. Never print the + // success tick here — that is the #5743 false success. + fmt.Fprintf(os.Stderr, "Bootstrap did not set up a database: %s\n", plan.Reason) + if plan.BlockedRemote != "" { + // Deliberately not "delete sync.remote and re-run": the git + // origin probe on that path swallows its error, so for a + // credential failure the retry would "succeed" by creating a + // fresh database that has diverged from the team's history. + fmt.Fprintf(os.Stderr, " Reproduce: git ls-remote %s refs/dolt/data\n", + gitRemoteURLForLsRemote(plan.BlockedRemote)) + fmt.Fprintf(os.Stderr, " If the remote is right, fix access (credentials, network, VPN) and re-run 'bd bootstrap'.\n") + fmt.Fprintf(os.Stderr, " If it is wrong, point sync.remote at the Dolt remote: bd dolt remote add origin \n") + } + return + } fmt.Printf("✓ Database already exists: %s\n", plan.BeadsDir) if !usesSQLServer() { fmt.Printf(" Nothing to do.\n") @@ -542,7 +743,7 @@ func printBootstrapPlan(plan BootstrapPlan) { } case "sync": fmt.Printf("Bootstrap plan: clone from remote\n") - fmt.Printf(" Remote: %s\n", plan.SyncRemote) + fmt.Printf(" Remote: %s\n", redactRemoteURL(plan.SyncRemote)) fmt.Printf(" Database: %s\n", plan.Database) case "restore": fmt.Printf("Bootstrap plan: restore from backup\n") @@ -643,10 +844,32 @@ func executeInitAction(ctx context.Context, plan BootstrapPlan, cfg *configfile. return fmt.Errorf("commit: %w", err) } + wireBootstrapDoltRemote(ctx, s, plan) fmt.Fprintf(os.Stderr, "Created fresh database with prefix %q\n", prefix) return nil } +// wireBootstrapDoltRemote registers plan.SyncRemote as Dolt remote "origin" on +// a workspace bootstrap just provisioned locally. +// +// The sync action has always done this (via configureInitDoltRemote after the +// clone) and bd init does it too, but the local actions never did, because a +// configured sync.remote used to short-circuit to "sync" before they could be +// reached. The refs/dolt/data fall-through makes them reachable, and without +// the remote the promise that "the first bd dolt push creates refs/dolt/data +// on that same git remote" breaks: push either refuses to adopt a remote +// without consent, or adopts the git origin — a different repository under the +// dedicated-data-repo pattern — and rewrites the committed sync.remote to +// match. Two machines bootstrapping in the pre-first-push window would then +// mint independent Dolt identities and diverge, which is the exact failure +// bootstrap exists to prevent (#5743). +func wireBootstrapDoltRemote(ctx context.Context, s storage.DoltStorage, plan BootstrapPlan) { + if plan.SyncRemote == "" { + return + } + configureInitDoltRemote(ctx, s, doltRemoteURL(plan.SyncRemote), false) +} + func executeRestoreAction(ctx context.Context, plan BootstrapPlan, cfg *configfile.Config) error { prefix := inferPrefix(cfg) dbName := cfg.GetDoltDatabase() @@ -674,6 +897,7 @@ func executeRestoreAction(ctx context.Context, plan BootstrapPlan, cfg *configfi return fmt.Errorf("restore from backup: %w", err) } + wireBootstrapDoltRemote(ctx, s, plan) fmt.Fprintf(os.Stderr, "Restored from backup\n") return nil } @@ -710,6 +934,7 @@ func executeJSONLImportAction(ctx context.Context, plan BootstrapPlan, cfg *conf return fmt.Errorf("commit import: %w", err) } + wireBootstrapDoltRemote(ctx, s, plan) fmt.Fprintf(os.Stderr, "Imported %d issues from %s\n", count, plan.JSONLFile) return nil } diff --git a/cmd/bd/bootstrap_guard_test.go b/cmd/bd/bootstrap_guard_test.go index f4f3e78ef..6a3df5560 100644 --- a/cmd/bd/bootstrap_guard_test.go +++ b/cmd/bd/bootstrap_guard_test.go @@ -3,8 +3,11 @@ package main import ( + "encoding/json" + "errors" "os" "path/filepath" + "strings" "testing" "time" @@ -92,46 +95,562 @@ func setupSyncRemoteConfig(t *testing.T, beadsDir, remote string) func() { return func() { config.ResetForTesting() } } -// TestDetectBootstrapAction_GitCodeRepoSyncRemoteBlocked verifies that a -// sync.remote pointing at a git code-repository host is rejected (action=none) -// instead of triggering DOLT_CLONE. -func TestDetectBootstrapAction_GitCodeRepoSyncRemoteBlocked(t *testing.T) { - for _, remote := range []string{ - "git+ssh://github.com/org/repo.git", - "https://github.com/org/repo.git", - "git@github.com:org/repo.git", - "https://gitlab.com/group/repo.git", - } { - t.Run(remote, func(t *testing.T) { - snapshotBootstrapEnv(t) +// stubProbeGitRemoteDoltData replaces the refs/dolt/data probe for the +// duration of a test. Every test that configures a forge sync.remote must +// install a stub: the real probe shells out to `git ls-remote` against +// whatever host the fixture URL names. +// +// A configured sync.remote also suppresses the git-origin auto-detect leg +// (gitOriginHasDoltDataRef, which is not behind this seam), so these tests +// cannot reach the network through that path either — including the +// fall-through cases, whose whole point is to keep going after the probe. +func stubProbeGitRemoteDoltData(t *testing.T, fn func(string) (bool, error)) { + t.Helper() + orig := probeGitRemoteDoltData + probeGitRemoteDoltData = fn + t.Cleanup(func() { probeGitRemoteDoltData = orig }) +} - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatal(err) +// forgeSyncRemoteForms enumerates the shapes a git-forge sync.remote arrives +// in, with the git+ URL normalizeRemoteURL produces for each. Every one of +// them is a value bd itself writes (bd init from a git origin, bd dolt remote +// add, a hand-edited config.yaml, or finalizeSyncedBootstrap). +var forgeSyncRemoteForms = []struct { + name string + remote string + wantClone string +}{ + {"raw https with .git", "https://github.com/org/repo.git", "git+https://github.com/org/repo.git"}, + {"raw https without .git", "https://github.com/org/repo", "git+https://github.com/org/repo"}, + {"scp style", "git@github.com:org/repo.git", "git+ssh://git@github.com/org/repo.git"}, + {"ssh scheme", "ssh://git@github.com/org/repo.git", "git+ssh://git@github.com/org/repo.git"}, + {"already git+https", "git+https://github.com/org/repo.git", "git+https://github.com/org/repo.git"}, + {"git+ssh without user", "git+ssh://github.com/org/repo.git", "git+ssh://github.com/org/repo.git"}, + {"gitlab", "https://gitlab.com/group/repo.git", "git+https://gitlab.com/group/repo.git"}, +} + +// newForgeSyncRemoteWorkspace builds a workspace whose config.yaml carries +// remote as sync.remote and chdirs into it, returning the .beads path. +func newForgeSyncRemoteWorkspace(t *testing.T, remote string) string { + t.Helper() + snapshotBootstrapEnv(t) + + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + cleanup := setupSyncRemoteConfig(t, beadsDir, remote) + t.Cleanup(cleanup) + + oldWd, _ := os.Getwd() + t.Cleanup(func() { _ = os.Chdir(oldWd) }) + if err := os.Chdir(tmpDir); err != nil { + t.Fatal(err) + } + return beadsDir +} + +// TestDetectBootstrapAction_GitForgeSyncRemote_ProbeRouting covers the #5743 / +// #5663 fix: a git-forge sync.remote is a supported Dolt remote, so bootstrap +// probes refs/dolt/data and routes on the answer instead of rejecting the URL +// outright (which printed a success tick and exited 0 with no database). +func TestDetectBootstrapAction_GitForgeSyncRemote_ProbeRouting(t *testing.T) { + t.Run("data present clones the git+ form", func(t *testing.T) { + for _, tc := range forgeSyncRemoteForms { + t.Run(tc.name, func(t *testing.T) { + beadsDir := newForgeSyncRemoteWorkspace(t, tc.remote) + + var probed []string + stubProbeGitRemoteDoltData(t, func(url string) (bool, error) { + probed = append(probed, url) + return true, nil + }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if plan.Action != "sync" { + t.Fatalf("action=%q, want %q (remote carries refs/dolt/data)", plan.Action, "sync") + } + if plan.SyncRemote != tc.wantClone { + t.Errorf("SyncRemote=%q, want %q", plan.SyncRemote, tc.wantClone) + } + // The durable #4421 pin: once isGitCodeRepoURL recognizes a + // URL as a forge, no raw http(s) form of it may reach + // DOLT_CLONE, because Dolt routes those through the + // remotesapi client and retries forever at ~1000% CPU. The + // git+ prefix is what selects the git remote factory. + // + // Scope note: this pins the routing, not the classifier. A + // forge the classifier does not recognize (self-hosted Gitea + // at https://git.example.com/org/repo, no .git suffix) still + // takes the trusted-as-is path unprobed, exactly as it did + // before this change — a pre-existing gap tracked separately. + if !strings.HasPrefix(plan.SyncRemote, "git+") { + t.Errorf("SyncRemote=%q lacks the git+ prefix; a recognized forge URL must never reach DOLT_CLONE raw (#4421)", plan.SyncRemote) + } + if plan.Blocked { + t.Error("Blocked=true on a successful sync plan") + } + if len(probed) != 1 || probed[0] != tc.wantClone { + t.Errorf("probe calls = %v, want exactly [%q]", probed, tc.wantClone) + } + }) + } + }) + + t.Run("no data falls through to init and still wires the remote", func(t *testing.T) { + beadsDir := newForgeSyncRemoteWorkspace(t, "https://github.com/org/repo.git") + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, nil }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if plan.Action != "init" { + t.Fatalf("action=%q, want %q (remote wired before the first bd dolt push)", plan.Action, "init") + } + if plan.Blocked { + t.Error("Blocked=true; a clean 'no data yet' probe is not a failure") + } + // Carried so executeInitAction can register it as Dolt "origin". + // Without it the first `bd dolt push` has no remote and either + // refuses or adopts the git origin — a different repository under + // the dedicated-data-repo pattern. + if plan.SyncRemote != "git+https://github.com/org/repo.git" { + t.Errorf("SyncRemote=%q, want the routed URL so the fresh database gets wired to it", plan.SyncRemote) + } + }) + + t.Run("no data falls through to backup restore", func(t *testing.T) { + beadsDir := newForgeSyncRemoteWorkspace(t, "https://github.com/org/repo.git") + backupDir := filepath.Join(beadsDir, "backup") + if err := os.MkdirAll(backupDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backupDir, "issues.jsonl"), []byte(`{"id":"bd-1"}`+"\n"), 0o644); err != nil { + t.Fatal(err) + } + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, nil }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + // Proves the fall-through runs the whole remaining detector chain, + // not just the fresh-init tail. + if plan.Action != "restore" { + t.Fatalf("action=%q, want %q (backup must still be found after the probe falls through)", plan.Action, "restore") + } + if plan.BackupDir != backupDir { + t.Errorf("BackupDir=%q, want %q", plan.BackupDir, backupDir) + } + }) + + // The error text is the shape the real probe now produces: git's own + // stderr, threaded out of exec.ExitError.Stderr by gitLsRemoteProbeError. + probeErr := errors.New("git ls-remote: exit status 128: fatal: Could not read from remote repository.") + + t.Run("probe error blocks and never clones", func(t *testing.T) { + const remote = "git+ssh://github.com/org/repo.git" + beadsDir := newForgeSyncRemoteWorkspace(t, remote) + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, probeErr }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if plan.Action != "none" { + t.Fatalf("action=%q, want %q (UNKNOWN must fail closed)", plan.Action, "none") + } + if !plan.Blocked { + t.Error("Blocked=false; an unverifiable remote must exit non-zero, not print a success tick") + } + if plan.SyncRemote != "" { + t.Errorf("SyncRemote=%q, want empty (an unverified URL must not be cloned from or pushed to)", plan.SyncRemote) + } + if plan.BlockedRemote != remote { + t.Errorf("BlockedRemote=%q, want %q (needed for the ls-remote hint)", plan.BlockedRemote, remote) + } + if !strings.Contains(plan.Reason, remote) { + t.Errorf("Reason=%q does not name the offending URL %q", plan.Reason, remote) + } + if !strings.Contains(plan.Reason, "Could not read from remote repository") { + t.Errorf("Reason=%q does not carry git's own diagnosis", plan.Reason) + } + }) + + t.Run("probe error still restores from a local backup", func(t *testing.T) { + // Offline laptop or credential-less CI: verifying the remote failed, + // but restoring a local backup touches no remote. The same machine + // with network and an empty remote would restore from this identical + // file, so a probe failure must not be the thing that denies it. + beadsDir := newForgeSyncRemoteWorkspace(t, "https://github.com/org/repo.git") + backupDir := filepath.Join(beadsDir, "backup") + if err := os.MkdirAll(backupDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backupDir, "issues.jsonl"), []byte(`{"id":"bd-1"}`+"\n"), 0o644); err != nil { + t.Fatal(err) + } + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, probeErr }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if plan.Action != "restore" { + t.Fatalf("action=%q reason=%q, want %q", plan.Action, plan.Reason, "restore") + } + if plan.Blocked { + t.Error("Blocked=true on a plan that recovers locally") + } + if plan.SyncRemote != "" { + t.Errorf("SyncRemote=%q, want empty: the remote is unverified, so nothing may be wired to it", plan.SyncRemote) + } + }) + + t.Run("probe error still imports git-tracked jsonl", func(t *testing.T) { + beadsDir := newForgeSyncRemoteWorkspace(t, "https://github.com/org/repo.git") + jsonl := filepath.Join(beadsDir, "issues.jsonl") + if err := os.WriteFile(jsonl, []byte(`{"id":"bd-1"}`+"\n"), 0o644); err != nil { + t.Fatal(err) + } + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, probeErr }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if plan.Action != "jsonl-import" { + t.Fatalf("action=%q reason=%q, want %q", plan.Action, plan.Reason, "jsonl-import") + } + if plan.Blocked { + t.Error("Blocked=true on a plan that recovers locally") + } + }) + + t.Run("credentials are stripped from the reason and the blocked URL", func(t *testing.T) { + const token = "ghp_SUPERSECRETTOKENVALUE" + beadsDir := newForgeSyncRemoteWorkspace(t, "https://x-access-token:"+token+"@github.com/org/repo.git") + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, probeErr }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if !plan.Blocked { + t.Fatalf("action=%q, want a blocked plan", plan.Action) + } + // Reason and blocked_remote both land in CI logs and agent + // transcripts; the clone funnel already scrubs userinfo before + // reporting and these diagnostics must not be the hole in that. + if strings.Contains(plan.Reason, token) { + t.Errorf("Reason leaks the token: %q", plan.Reason) + } + if strings.Contains(plan.BlockedRemote, token) { + t.Errorf("BlockedRemote leaks the token: %q", plan.BlockedRemote) + } + if !strings.Contains(plan.BlockedRemote, "github.com/org/repo.git") { + t.Errorf("BlockedRemote=%q lost the part the user needs", plan.BlockedRemote) + } + }) + + t.Run("routing is cwd independent in a monorepo subdir", func(t *testing.T) { + remote := "https://github.com/org/repo.git" + beadsDir := newForgeSyncRemoteWorkspace(t, remote) + + subDir := filepath.Join(filepath.Dir(beadsDir), "sub", "dir") + if err := os.MkdirAll(subDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chdir(subDir); err != nil { + t.Fatal(err) + } + + var probed []string + stubProbeGitRemoteDoltData(t, func(url string) (bool, error) { + probed = append(probed, url) + return true, nil + }) + + plan := detectBootstrapAction(beadsDir, configfile.DefaultConfig()) + + if plan.Action != "sync" { + t.Fatalf("action=%q, want %q from a subdirectory", plan.Action, "sync") + } + if len(probed) != 1 || probed[0] != "git+https://github.com/org/repo.git" { + t.Errorf("probe calls = %v, want exactly [%q]", probed, "git+https://github.com/org/repo.git") + } + }) +} + +// TestBootstrapPlanOutcome pins the exit status of every plan shape. +// Action=="none" used to be an unconditional success, which is how a rejected +// sync.remote exited 0 with no database (#5743). Blocked is the single +// discriminator; --dry-run is always a successful preview. +func TestBootstrapPlanOutcome(t *testing.T) { + tests := []struct { + name string + plan BootstrapPlan + dryRun bool + wantExit int // 0 means "nil error" + }{ + { + name: "existing database is a genuine no-op", + plan: BootstrapPlan{Action: "none", HasExisting: true, Reason: "Database already exists at /tmp/x"}, + wantExit: 0, + }, + { + name: "blocked plan fails", + plan: BootstrapPlan{Action: "none", Blocked: true, Reason: "could not verify refs/dolt/data"}, + wantExit: 1, + }, + { + name: "sync plan proceeds", + plan: BootstrapPlan{Action: "sync", SyncRemote: "git+https://github.com/org/repo.git"}, + wantExit: 0, + }, + { + name: "init plan proceeds", + plan: BootstrapPlan{Action: "init"}, + wantExit: 0, + }, + { + // bd doctor documents `bd bootstrap --dry-run` as the safe + // inspection step, so a preview must never abort a set -e script. + name: "dry run previews a blocked plan without failing", + plan: BootstrapPlan{Action: "none", Blocked: true, Reason: "could not verify refs/dolt/data"}, + dryRun: true, + wantExit: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := bootstrapPlanOutcome(tt.plan, tt.dryRun) + if tt.wantExit == 0 { + if err != nil { + t.Fatalf("bootstrapPlanOutcome() = %v, want nil", err) + } + return } - cleanup := setupSyncRemoteConfig(t, beadsDir, remote) - defer cleanup() - - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - if err := os.Chdir(tmpDir); err != nil { - t.Fatal(err) + code, ok := exitCodeFromError(err) + if !ok { + t.Fatalf("bootstrapPlanOutcome() = %v, want an *exitError", err) } - - cfg := configfile.DefaultConfig() - plan := detectBootstrapAction(beadsDir, cfg) - - if plan.Action != "none" { - t.Errorf("remote=%q: action=%q, want %q (git code-repo URL must be blocked)", remote, plan.Action, "none") - } - if plan.SyncRemote != "" { - t.Errorf("remote=%q: SyncRemote=%q, want empty (blocked URL must not propagate)", remote, plan.SyncRemote) + if code != tt.wantExit { + t.Errorf("exit code = %d, want %d", code, tt.wantExit) } }) } } +// TestDetectBootstrapAction_BlockedIsDerivedNotInherited covers the state-leak +// class the seeded plan used to create: existingBootstrapDBPlan's "none" +// verdict was copied into the working plan, so every later branch inherited +// its HasExisting/Blocked/Reason. A probe error on top of a seeded +// HasExisting=true produced action=none + blocked=true + has_existing=true, +// which printed the success tick and exited 0 — the #5743 bug, reintroduced. +func TestDetectBootstrapAction_BlockedIsDerivedNotInherited(t *testing.T) { + // newSeededServerWorkspace builds the synthesized-beadsDir shape: the + // .beads directory does not exist, so existingBootstrapDBPlan's verdict is + // held as a fallback rather than returned outright. + newSeededServerWorkspace := func(t *testing.T, remote string, check bootstrapServerDBCheck) (string, *configfile.Config) { + t.Helper() + snapshotBootstrapEnv(t) + + tmpDir := t.TempDir() + configDir := filepath.Join(tmpDir, "config-beads") + if err := os.MkdirAll(configDir, 0o750); err != nil { + t.Fatal(err) + } + cleanup := setupSyncRemoteConfig(t, configDir, remote) + t.Cleanup(cleanup) + + doltDataDir := filepath.Join(tmpDir, "dolt-data") + if err := os.MkdirAll(filepath.Join(doltDataDir, "mydb"), 0o750); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DOLT_DATA_DIR", doltDataDir) + + oldWd, _ := os.Getwd() + t.Cleanup(func() { _ = os.Chdir(oldWd) }) + if err := os.Chdir(tmpDir); err != nil { + t.Fatal(err) + } + + origCheck := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { return check } + t.Cleanup(func() { checkBootstrapServerDB = origCheck }) + + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + t.Cleanup(func() { bootstrapRetryDelay = origDelay }) + + cfg := configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "mydb" + cfg.DoltDataDir = doltDataDir + // Never created: this is the "fresh clone, .beads synthesized" path. + return filepath.Join(tmpDir, ".beads"), cfg + } + + t.Run("existing server database wins over a failed probe", func(t *testing.T) { + beadsDir, cfg := newSeededServerWorkspace(t, "https://github.com/org/repo.git", + bootstrapServerDBCheck{Exists: true, Reachable: true}) + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { + return false, errors.New("git ls-remote: exit status 128") + }) + + plan := detectBootstrapAction(beadsDir, cfg) + + if !plan.HasExisting { + t.Fatalf("HasExisting=false; the server database exists (action=%q reason=%q)", plan.Action, plan.Reason) + } + if plan.Blocked { + t.Error("Blocked=true alongside HasExisting=true — self-contradictory, and it would print the tick AND claim a block") + } + if err := bootstrapPlanOutcome(plan, false); err != nil { + t.Errorf("outcome = %v, want nil: a database exists", err) + } + }) + + t.Run("failed server verification does not taint a later sync plan", func(t *testing.T) { + beadsDir, cfg := newSeededServerWorkspace(t, "https://github.com/org/repo.git", + bootstrapServerDBCheck{Reachable: true, Err: errors.New("dial tcp: connection refused")}) + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return true, nil }) + + plan := detectBootstrapAction(beadsDir, cfg) + + if plan.Action != "sync" { + t.Fatalf("action=%q reason=%q, want %q", plan.Action, plan.Reason, "sync") + } + if plan.Blocked { + t.Error(`Blocked=true on a sync plan: "nothing may clone from a blocked plan" is the field's own contract`) + } + if plan.BlockedRemote != "" { + t.Errorf("BlockedRemote=%q on a sync plan", plan.BlockedRemote) + } + }) + + t.Run("unverifiable server database with no other source blocks", func(t *testing.T) { + beadsDir, cfg := newSeededServerWorkspace(t, "", + bootstrapServerDBCheck{Reachable: true, Err: errors.New("dial tcp: connection refused")}) + + plan := detectBootstrapAction(beadsDir, cfg) + + if plan.Action != "none" || !plan.Blocked { + t.Fatalf("action=%q blocked=%v, want none+blocked", plan.Action, plan.Blocked) + } + if err := bootstrapPlanOutcome(plan, false); err == nil { + t.Error("outcome = nil; an unverifiable database must not report success") + } + }) +} + +// TestPrintBootstrapPlan_NoneWithoutDB_NoSuccessTick pins the other half of +// the #5743 false success: the "✓ Database already exists" line must be +// reserved for plans that actually found a database. +func TestPrintBootstrapPlan_NoneWithoutDB_NoSuccessTick(t *testing.T) { + blocked := BootstrapPlan{ + Action: "none", + Blocked: true, + BeadsDir: "/workspace/.beads", + BlockedRemote: "git+ssh://github.com/org/repo.git", + Reason: `could not verify refs/dolt/data on sync.remote "git+ssh://github.com/org/repo.git": git ls-remote: exit status 128: fatal: Could not read from remote repository.`, + } + + t.Run("blocked plan prints nothing to stdout", func(t *testing.T) { + out := captureStdout(t, func() error { printBootstrapPlan(blocked); return nil }) + if strings.TrimSpace(out) != "" { + t.Fatalf("blocked plan wrote to stdout (the success tick lives there):\n%s", out) + } + }) + + t.Run("blocked plan reports the reason and actionable hints on stderr", func(t *testing.T) { + out := captureStderr(t, func() { printBootstrapPlan(blocked) }) + + if strings.Contains(out, "Database already exists") { + t.Fatalf("blocked plan printed a success tick:\n%s", out) + } + if !strings.Contains(out, blocked.Reason) { + t.Errorf("output does not carry the reason:\n%s", out) + } + // The hint must be runnable: git ls-remote does not understand the + // git+ prefix, so it is stripped exactly as the probe strips it. + if !strings.Contains(out, "git ls-remote ssh://github.com/org/repo.git refs/dolt/data") { + t.Errorf("output lacks a runnable ls-remote hint:\n%s", out) + } + // Deleting sync.remote is NOT offered as a remedy: the git-origin + // probe swallows its error, so on a credential failure that retry + // would "succeed" by creating a database diverged from the team's. + if strings.Contains(out, "remove 'sync.remote'") { + t.Errorf("hint still suggests deleting sync.remote, which can silently diverge:\n%s", out) + } + }) + + t.Run("existing database keeps the success tick", func(t *testing.T) { + plan := BootstrapPlan{ + Action: "none", + HasExisting: true, + BeadsDir: "/workspace/.beads", + Reason: "Database already exists at /workspace/.beads/embeddeddolt", + } + + out := captureStdout(t, func() error { printBootstrapPlan(plan); return nil }) + + if !strings.Contains(out, "✓ Database already exists: /workspace/.beads") { + t.Fatalf("existing database lost its success line:\n%s", out) + } + if !strings.Contains(out, "Nothing to do") { + t.Errorf("existing database lost its 'Nothing to do' line:\n%s", out) + } + }) + + t.Run("sync plan does not print credentials", func(t *testing.T) { + const token = "ghp_SUPERSECRETTOKENVALUE" + plan := BootstrapPlan{ + Action: "sync", + BeadsDir: "/workspace/.beads", + Database: "beads", + SyncRemote: "git+https://x-access-token:" + token + "@github.com/org/repo.git", + } + + out := captureStdout(t, func() error { printBootstrapPlan(plan); return nil }) + + if strings.Contains(out, token) { + t.Fatalf("plan output leaks the token:\n%s", out) + } + if !strings.Contains(out, "github.com/org/repo.git") { + t.Errorf("plan output lost the remote entirely:\n%s", out) + } + }) +} + +// TestBootstrapPlanJSON_RedactsCredentials closes the JSON half of the same +// credential leak: `bd bootstrap --json` serializes the plan, and BootstrapPlan. +// MarshalJSON must redact SyncRemote the way printBootstrapPlan redacts the human +// path. The existing "sync plan does not print credentials" subtest covers only +// printBootstrapPlan, so without this the JSON gap is invisible. It also pins the +// "redact on a copy" contract — the plan executeBootstrapPlan clones from must +// keep its real credentials. +func TestBootstrapPlanJSON_RedactsCredentials(t *testing.T) { + const token = "ghp_SUPERSECRETTOKENVALUE" + plan := BootstrapPlan{ + Action: "sync", + BeadsDir: "/workspace/.beads", + Database: "beads", + SyncRemote: "git+https://x-access-token:" + token + "@github.com/org/repo.git", + } + + encoded, err := json.Marshal(plan) + if err != nil { + t.Fatalf("json.Marshal(plan) failed: %v", err) + } + out := string(encoded) + + if strings.Contains(out, token) { + t.Fatalf("bootstrap plan JSON leaks the token:\n%s", out) + } + if !strings.Contains(out, "github.com/org/repo.git") { + t.Errorf("bootstrap plan JSON dropped the remote host/path entirely:\n%s", out) + } + // Redaction must not mutate the plan executeBootstrapPlan consumes for the + // clone, or the clone would lose the credentials it needs. + if !strings.Contains(plan.SyncRemote, token) { + t.Errorf("MarshalJSON mutated the source plan's SyncRemote: %q", plan.SyncRemote) + } +} + // TestDetectBootstrapAction_ValidDoltSyncRemoteUnchanged verifies that a valid // Dolt remote URL is still accepted as-is (no regression from Layer 1 guard). func TestDetectBootstrapAction_ValidDoltSyncRemoteUnchanged(t *testing.T) { @@ -141,21 +660,15 @@ func TestDetectBootstrapAction_ValidDoltSyncRemoteUnchanged(t *testing.T) { "git+ssh://my-self-hosted-dolt.example.com/org/db", } { t.Run(remote, func(t *testing.T) { - snapshotBootstrapEnv(t) + beadsDir := newForgeSyncRemoteWorkspace(t, remote) - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatal(err) - } - cleanup := setupSyncRemoteConfig(t, beadsDir, remote) - defer cleanup() - - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - if err := os.Chdir(tmpDir); err != nil { - t.Fatal(err) - } + // Non-forge remotes take the trusted-as-is path and must never + // pay for an ls-remote — a Dolt remotesapi endpoint has no + // refs/dolt/data to probe in the first place. + stubProbeGitRemoteDoltData(t, func(url string) (bool, error) { + t.Fatalf("probed non-forge remote %q; the trusted-as-is path must not probe", url) + return false, nil + }) cfg := configfile.DefaultConfig() plan := detectBootstrapAction(beadsDir, cfg) diff --git a/cmd/bd/init.go b/cmd/bd/init.go index f87a0add2..c5fb383ac 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -1279,6 +1279,10 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } } if syncFromRemote { + // A raw https:// forge URL would route through Dolt's remotesapi + // client and spin forever (#4421); its git+ form routes through + // the git remote factory. Non-forge URLs are untouched (GH#3339). + syncURL = doltRemoteURL(syncURL) cloneCfg := initTimeCloneConfig(initServerMode, serverHost, serverPort, serverSocket, serverUser, dbName) disposition, err := runInitRemoteClone(syncURL, func(remoteURL string) error { return cloneFromRemoteWithMode(ctx, beadsDir, remoteURL, dbName, cloneCfg, initRemoteCloneMode(initServerMode, externalServer)) @@ -1494,8 +1498,14 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // falling back to JSONL-as-sync. Gateway mode skips it: the hosted // server owns the database, so DOLT_REMOTE('add', ...) must not run // against it (see shouldWriteInitDoltRemote). + // The URL is routed here too, not just on the clone path: the + // no-Dolt-data-yet case (the pre-first-push wiring this is for) never + // clones, so without routing a raw https forge URL would be stored + // verbatim in dolt_remotes and the *first* `bd dolt push` would take + // it to the remotesapi client — the #4421 storm, one leg further down + // (#5743). if shouldWriteInitDoltRemote(doltCfg.Gateway, syncURL, syncFromRemote, syncURLFromConfig, syncURLFromGitOrigin, isDoltLocalOnly()) { - configureInitDoltRemote(ctx, store, syncURL, quiet) + configureInitDoltRemote(ctx, store, doltRemoteURL(syncURL), quiet) } // === CONFIGURATION METADATA (Pattern A: Fatal) === @@ -1730,7 +1740,9 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // Must run AFTER createConfigYaml which creates the file. // Persist the effective remote so future bootstrap and hooks know // Dolt, not JSONL, is the cross-machine sync path. Plain git - // origins are valid here: the first push creates refs/dolt/data. + // origins are valid here: the first push creates refs/dolt/data, + // and bootstrap resolves such a URL by probing refs/dolt/data + // rather than rejecting it (#5743). if err := persistInitSyncRemote(beadsDir, initRemote, syncURL, syncFromRemote, syncURLFromConfig, syncURLFromGitOrigin); err != nil { return fmt.Errorf("failed to persist sync.remote to config.yaml: %v", err) } diff --git a/cmd/bd/init_policy_test.go b/cmd/bd/init_policy_test.go index 273a0bdab..4db77eb49 100644 --- a/cmd/bd/init_policy_test.go +++ b/cmd/bd/init_policy_test.go @@ -192,3 +192,61 @@ func TestNormalizeIssuePrefixAndDatabaseName(t *testing.T) { }) } } + +// TestDoltRemoteURL pins which URLs bd rewrites before handing them to Dolt — +// for DOLT_CLONE, for DOLT_REMOTE('add', ...) and therefore for DOLT_PUSH. +// Forge URLs must arrive in git+ form so Dolt uses the git remote factory +// rather than the remotesapi retry storm (#4421); a user-configured remotesapi +// endpoint must arrive byte-identical (GH#3339). bd init and bd bootstrap +// share this one helper so they can never disagree about the URL derived from +// a single committed sync.remote. +func TestDoltRemoteURL(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"raw https forge", "https://github.com/org/repo.git", "git+https://github.com/org/repo.git"}, + {"forge without .git", "https://github.com/org/repo", "git+https://github.com/org/repo"}, + {"scp style forge", "git@github.com:org/repo.git", "git+ssh://git@github.com/org/repo.git"}, + {"ssh forge", "ssh://git@gitlab.com/org/repo.git", "git+ssh://git@gitlab.com/org/repo.git"}, + {"already normalized", "git+https://github.com/org/repo.git", "git+https://github.com/org/repo.git"}, + + {"remotesapi endpoint", "http://myserver:7007/mydb", "http://myserver:7007/mydb"}, + {"remotesapi https endpoint", "https://doltremoteapi.dolthub.com/org/db", "https://doltremoteapi.dolthub.com/org/db"}, + {"dolthub native", "dolthub://myorg/mydb", "dolthub://myorg/mydb"}, + {"s3 native", "s3://bucket/path", "s3://bucket/path"}, + {"self-hosted dolt over ssh", "git+ssh://my-dolt.example.com/org/db", "git+ssh://my-dolt.example.com/org/db"}, + {"empty", "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := doltRemoteURL(tt.in); got != tt.want { + t.Errorf("doltRemoteURL(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +// TestRunInitRemoteCloneReceivesRoutedURL proves the routing survives the call +// into runInitRemoteClone — the seam the init RunE body actually uses. +func TestRunInitRemoteCloneReceivesRoutedURL(t *testing.T) { + for _, tt := range []struct{ in, want string }{ + {"https://github.com/org/repo.git", "git+https://github.com/org/repo.git"}, + {"http://myserver:7007/mydb", "http://myserver:7007/mydb"}, + } { + t.Run(tt.in, func(t *testing.T) { + var gotURL string + if _, err := runInitRemoteClone(doltRemoteURL(tt.in), func(url string) error { + gotURL = url + return nil + }); err != nil { + t.Fatalf("runInitRemoteClone: %v", err) + } + if gotURL != tt.want { + t.Errorf("clone received %q, want %q", gotURL, tt.want) + } + }) + } +} diff --git a/cmd/bd/init_safety_test.go b/cmd/bd/init_safety_test.go index 846266ff8..8119006fe 100644 --- a/cmd/bd/init_safety_test.go +++ b/cmd/bd/init_safety_test.go @@ -358,6 +358,12 @@ func TestShouldWireInitRemote(t *testing.T) { want: false, }, { + // Keep this true. The URL is the supported pre-first-push wiring: + // the first `bd dolt push` creates refs/dolt/data on that same git + // remote, and bootstrap resolves the persisted value by probing + // for that ref (#5743). Do not "fix" a bootstrap failure by + // un-persisting here — that leaves every already-committed + // .beads/config.yaml in the wild broken. name: "plain git origin without dolt data", syncURL: "git+https://github.com/org/plain-source.git", syncURLFromGit: true, diff --git a/cmd/bd/sync_git.go b/cmd/bd/sync_git.go index 84ae162fd..ee42fe8b4 100644 --- a/cmd/bd/sync_git.go +++ b/cmd/bd/sync_git.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/exec" @@ -112,18 +113,60 @@ func gitOriginHasDoltDataRefStatus() (bool, error) { } // A non-nil error means UNKNOWN, not "no data" — the bool is meaningless. +// +// The error names the failure, not the remote: every caller already names the +// remote it probed, and wrapping here too produced messages that said the URL +// and the ref twice ("...on sync.remote \"X\": probe refs/dolt/data on X: exit +// status 128"). func gitRemoteHasDoltDataRefStatus(remote string) (bool, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), gitDoltDataProbeTimeout) defer cancel() cmd := exec.CommandContext(ctx, "git", "ls-remote", gitRemoteURLForLsRemote(remote), "refs/dolt/data") cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") output, err := cmd.Output() if err != nil { - return false, fmt.Errorf("probe refs/dolt/data on %s: %w", remote, err) + return false, gitLsRemoteProbeError(ctx, err) } return strings.TrimSpace(string(output)) != "", nil } +const gitDoltDataProbeTimeout = 10 * time.Second + +// gitLsRemoteProbeError turns a failed `git ls-remote` into something a user +// can act on. exec.ExitError.Error() is only "exit status 128" — git's actual +// complaint (auth vs. DNS vs. deleted repo) sits unread in ExitError.Stderr +// because cmd.Output() captures it there, and a context timeout surfaces as +// the equally opaque "signal: killed". +func gitLsRemoteProbeError(ctx context.Context, err error) error { + if ctx.Err() == context.DeadlineExceeded { + return fmt.Errorf("git ls-remote did not answer within %s (network, or an SSH key that wants a passphrase)", gitDoltDataProbeTimeout) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + // git can echo the remote it failed to reach — including a CI token in + // https://x-access-token:@host form — so scrub credentials before + // this detail reaches plan.Reason, which flows to both stderr and + // `bd bootstrap --json`. exitErr itself is only "exit status 128". + if detail := scrubURLCredentials(firstNonEmptyLine(string(exitErr.Stderr))); detail != "" { + return fmt.Errorf("git ls-remote: %v: %s", exitErr, detail) + } + } + return fmt.Errorf("git ls-remote: %w", err) +} + +// firstNonEmptyLine returns the first meaningful line of git's stderr. git +// prefixes most failures with a "remote:"/"fatal:" line that already says +// everything; the rest is usually a hint block we do not want inside a +// single-line Reason. +func firstNonEmptyLine(s string) string { + for _, line := range strings.Split(s, "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + return trimmed + } + } + return "" +} + func gitRemoteURLForLsRemote(remote string) string { return strings.TrimPrefix(remote, "git+") } diff --git a/cmd/bd/sync_git_test.go b/cmd/bd/sync_git_test.go index feb7e8d42..4294beb78 100644 --- a/cmd/bd/sync_git_test.go +++ b/cmd/bd/sync_git_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" ) @@ -195,3 +196,29 @@ func runGitForSyncTest(t *testing.T, dir string, args ...string) { t.Fatalf("git %v failed: %v\n%s", args, err, string(output)) } } + +// TestGitRemoteHasDoltDataRefStatusSurfacesGitStderr proves the probe failure a +// user actually sees carries git's own diagnosis. exec.ExitError.Error() is +// only "exit status 128"; git's complaint sits unread in ExitError.Stderr, and +// bootstrap's blocked message is built from this error (#5743). +func TestGitRemoteHasDoltDataRefStatusSurfacesGitStderr(t *testing.T) { + missing := filepath.Join(t.TempDir(), "not-a-repo") + + hasData, err := gitRemoteHasDoltDataRefStatus(missing) + + if err == nil { + t.Fatalf("probe of %q returned hasData=%v and no error; want a failure", missing, hasData) + } + msg := err.Error() + if !strings.Contains(msg, "ls-remote") { + t.Errorf("error %q does not name the command that failed", msg) + } + // Whatever git's exact wording, the message must be more than the bare + // exit status — that is the whole point of reading ExitError.Stderr. + if msg == "git ls-remote: exit status 128" { + t.Errorf("error %q carries no stderr detail; the ExitError.Stderr thread-through regressed", msg) + } + if !strings.Contains(msg, "repository") && !strings.Contains(msg, "fatal") { + t.Errorf("error %q does not look like it carries git's stderr", msg) + } +} diff --git a/cmd/bd/sync_remote.go b/cmd/bd/sync_remote.go index 6efdec3ab..4c8a1f86f 100644 --- a/cmd/bd/sync_remote.go +++ b/cmd/bd/sync_remote.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "regexp" "strings" "github.com/steveyegge/beads/internal/beads" @@ -66,3 +67,80 @@ func commitBeadsConfigForActiveRepo(ctx context.Context, msg string) { func normalizeRemoteURL(url string) string { return doltremote.Normalize(url) } + +// doltRemoteURL returns the form of remote that bd hands to Dolt — for +// DOLT_CLONE, for DOLT_REMOTE('add', ...), and therefore for DOLT_PUSH. +// +// A git-forge URL must reach Dolt in its git+ form. Dolt's dbfactory routes by +// scheme: raw http(s):// goes to the remotesapi client, which speaks Dolt's +// wire protocol at github.com and retries indefinitely (#4421), while +// git+https/git+ssh goes to the git remote factory, which shells out to git +// and fails cleanly. +// +// Everything else is returned byte-identical. That preserves GH#3339: a +// user-configured Dolt remotesapi endpoint (http://myserver:7007/mydb) must +// never be rewritten to git+http://, and it never classifies as a forge URL. +// +// This is the single owner of that routing rule. bd init and bd bootstrap must +// both call it: if they disagree about the URL derived from one committed +// sync.remote, that is the #5743 class of skew all over again. +func doltRemoteURL(remote string) string { + if isGitCodeRepoURL(remote) { + return normalizeRemoteURL(remote) + } + return remote +} + +// redactRemoteURL strips credentials from a remote URL so it is safe to echo +// into errors, hints, logs and JSON. CI commonly configures sync.remote as +// https://x-access-token:@github.com/org/repo.git, and the clone funnel +// already scrubs userinfo before reporting (versioncontrolops.sanitizeURL); +// diagnostics must not be the hole in that convention. +// +// HTTP(S) userinfo is transport credentials and is dropped whole. SSH userinfo +// selects the remote account (git@host is not a secret and is needed for the +// hint to be runnable), so only a password component is dropped. +func redactRemoteURL(raw string) string { + sep := strings.Index(raw, "://") + if sep < 0 { + // scp-style (git@host:path) or a bare path: no place for a password. + return raw + } + scheme, rest := raw[:sep], raw[sep+3:] + authority, tail := rest, "" + if slash := strings.Index(rest, "/"); slash >= 0 { + authority, tail = rest[:slash], rest[slash:] + } + at := strings.LastIndex(authority, "@") + if at < 0 { + return raw + } + userinfo, host := authority[:at], authority[at+1:] + if isDoltSSHScheme(scheme) { + if colon := strings.Index(userinfo, ":"); colon >= 0 { + userinfo = userinfo[:colon] + } + return scheme + "://" + userinfo + "@" + host + tail + } + return scheme + "://" + host + tail +} + +func isDoltSSHScheme(scheme string) bool { + return scheme == "ssh" || scheme == "git+ssh" +} + +// urlWithUserinfoRe matches the "scheme://userinfo@host" span of a URL embedded +// anywhere in free-form text — for example the git stderr line bootstrap folds +// into a probe error, which can echo the very remote (credentials and all) that +// git failed to reach. The userinfo run stops at the first "@" and the host run +// at the first "/" so a trailing path is left for redactRemoteURL to keep. +var urlWithUserinfoRe = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^@\s'"]*@[^/\s'"]*`) + +// scrubURLCredentials redacts credential-bearing URLs embedded anywhere in s by +// running each match through redactRemoteURL, so the same rule applied to the +// plan's own URL fields (http(s) drops userinfo, ssh drops only a password) also +// covers URLs that arrive inside arbitrary text. Text with no "userinfo@" URL is +// returned unchanged, so credential-free git diagnostics keep their full detail. +func scrubURLCredentials(s string) string { + return urlWithUserinfoRe.ReplaceAllStringFunc(s, redactRemoteURL) +} diff --git a/cmd/bd/sync_remote_test.go b/cmd/bd/sync_remote_test.go index bbbece6e9..5f9aeda02 100644 --- a/cmd/bd/sync_remote_test.go +++ b/cmd/bd/sync_remote_test.go @@ -131,3 +131,109 @@ func runGitForCommitConfigTest(t *testing.T, dir string, args ...string) string func shellQuoteForTest(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" } + +// TestRedactRemoteURL pins the credential scrubbing applied to every remote URL +// bd echoes into an error, a hint, a log line or JSON. CI commonly configures +// sync.remote as https://x-access-token:@github.com/org/repo.git. +func TestRedactRemoteURL(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + name: "https token is dropped whole", + in: "https://x-access-token:ghp_secret@github.com/org/repo.git", + want: "https://github.com/org/repo.git", + }, + { + name: "git+https token is dropped whole", + in: "git+https://user:pass@github.com/org/repo.git", + want: "git+https://github.com/org/repo.git", + }, + { + name: "http basic auth is dropped whole", + in: "http://user:pass@myserver:7007/mydb", + want: "http://myserver:7007/mydb", + }, + { + // The conventional SSH account selector is not a secret, and the + // ls-remote hint is useless without it. + name: "ssh user is kept", + in: "git+ssh://git@github.com/org/repo.git", + want: "git+ssh://git@github.com/org/repo.git", + }, + { + name: "ssh password is dropped, user kept", + in: "ssh://alice:hunter2@my-dolt.example.com/org/db", + want: "ssh://alice@my-dolt.example.com/org/db", + }, + { + name: "scp style is untouched", + in: "git@github.com:org/repo.git", + want: "git@github.com:org/repo.git", + }, + {name: "no userinfo", in: "https://github.com/org/repo.git", want: "https://github.com/org/repo.git"}, + {name: "dolt native", in: "dolthub://myorg/mydb", want: "dolthub://myorg/mydb"}, + {name: "empty", in: "", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := redactRemoteURL(tt.in); got != tt.want { + t.Errorf("redactRemoteURL(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +// TestScrubURLCredentials covers the redaction of credential-bearing URLs that +// arrive embedded in free-form text — most importantly a `git ls-remote` stderr +// line that echoes the tokenized remote. Credential-free diagnostics must pass +// through untouched so the probe error keeps git's actual complaint (#5743). +func TestScrubURLCredentials(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + // The contingent leak Finding 2 guards: a git build that echoes the + // full tokenized URL. The token must go; host and path must stay. + name: "tokenized git stderr is scrubbed, host and path kept", + in: "fatal: unable to access 'https://x-access-token:ghp_TESTTOKEN@github.com/org/repo.git/': The requested URL returned error: 403", + want: "fatal: unable to access 'https://github.com/org/repo.git/': The requested URL returned error: 403", + }, + { + name: "ssh password embedded in text is dropped, user kept", + in: "note: not cloning ssh://alice:hunter2@dolt.example.com/org/db right now", + want: "note: not cloning ssh://alice@dolt.example.com/org/db right now", + }, + { + name: "two credential URLs in one line are both scrubbed", + in: "https://u:p@a.example.com/x and https://v:q@b.example.com/y", + want: "https://a.example.com/x and https://b.example.com/y", + }, + { + // The common case: modern git already prints host-only, and DNS/repo + // errors carry no userinfo. Nothing must be altered. + name: "credential-free git stderr is unchanged", + in: "fatal: repository 'https://github.com/org/repo.git/' not found", + want: "fatal: repository 'https://github.com/org/repo.git/' not found", + }, + { + name: "plain diagnostic without a URL is unchanged", + in: "fatal: Could not read from remote repository.", + want: "fatal: Could not read from remote repository.", + }, + {name: "empty", in: "", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := scrubURLCredentials(tt.in); got != tt.want { + t.Errorf("scrubURLCredentials(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +}