diff --git a/internal/storage/dolt/bootstrap.go b/internal/storage/dolt/bootstrap.go index 054961f3f..227fa7e55 100644 --- a/internal/storage/dolt/bootstrap.go +++ b/internal/storage/dolt/bootstrap.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/steveyegge/beads/internal/configfile" @@ -13,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/gittraceenv" "github.com/steveyegge/beads/internal/lockfile" "github.com/steveyegge/beads/internal/remotecache" + "github.com/steveyegge/beads/internal/storage/doltutil" ) // staleLockAge is the maximum age of a lock file before it's considered stale. @@ -46,6 +48,21 @@ func BootstrapFromGitRemote(ctx context.Context, doltDir, gitRemoteURL string) ( // callers should use cfg.GetDoltDatabase() which applies the fallback chain // (env var → config → default). func BootstrapFromRemoteWithDB(ctx context.Context, doltDir, remoteURL, database string) (bool, error) { + return BootstrapFromRemoteWithDBRef(ctx, doltDir, remoteURL, database, "") +} + +// BootstrapFromRemoteWithDBRef is BootstrapFromRemoteWithDB for a git-backed +// remote whose Dolt data lives on the git ref ref (`dolt clone --ref`). An +// empty ref is Dolt's default, refs/dolt/data. +func BootstrapFromRemoteWithDBRef(ctx context.Context, doltDir, remoteURL, database, ref string) (bool, error) { + // The ref is checked before the already-exists early return so the same + // --ref argument is accepted or refused the same way on every host, rather + // than surfacing a typo only on a machine that has not bootstrapped yet. + // The URL and database checks keep their existing position below it. + if err := doltutil.ValidateGitDataRefArg(strings.TrimSpace(ref)); err != nil { + return false, fmt.Errorf("invalid git data ref: %w", err) + } + // Skip if Dolt database already exists if doltExists(doltDir) { return false, nil @@ -78,7 +95,7 @@ func BootstrapFromRemoteWithDB(ctx context.Context, doltDir, remoteURL, database // from an earlier bootstrap that a stale/empty doltExists() check // missed) that we must not delete. targetPreExisted := pathExists(cloneTarget) - cmd := bootstrapCloneCmd(ctx, remoteURL, cloneTarget) + cmd := bootstrapCloneCmd(ctx, remoteURL, cloneTarget, ref) if output, err := cmd.CombinedOutput(); err != nil { if targetPreExisted { return false, fmt.Errorf("dolt clone failed: %w\nOutput: %s\nClone target %q already existed before this attempt; left untouched to avoid deleting a pre-existing Dolt repo", err, output, cloneTarget) @@ -94,8 +111,8 @@ func BootstrapFromRemoteWithDB(ctx context.Context, doltDir, remoteURL, database // bootstrapCloneCmd builds the `dolt clone` for remote bootstrap. It does not // route through prepareDoltCLITransferCommand, so it applies the remote env // guards itself (see internal/gittraceenv and internal/githooksenv). -func bootstrapCloneCmd(ctx context.Context, remoteURL, cloneTarget string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "dolt", doltCloneArgs(remoteURL, cloneTarget)...) +func bootstrapCloneCmd(ctx context.Context, remoteURL, cloneTarget, ref string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "dolt", doltCloneArgs(remoteURL, cloneTarget, ref)...) // #nosec G204 -- validated remote URL; the ref is an option value, taken verbatim by dolt's argparser and checked by doltutil.ValidateGitDataRefArg cmd.Env = githooksenv.DisabledEnv(gittraceenv.ScrubEnv(os.Environ())) return cmd } @@ -144,11 +161,17 @@ func formatFailedCloneTargetError(cloneErr error, output []byte, cloneTarget str return fmt.Errorf("dolt clone failed: %w\nOutput: %s\nCould not clean up failed clone target %q after retrying: %v\nOn Windows this usually means a dolt or bd process, or antivirus scanner, still has a file handle open under `.dolt/noms/LOCK`. Stop stuck dolt/bd processes, wait a moment, delete the directory manually if it remains, then retry `bd bootstrap`", cloneErr, output, cloneTarget, cleanupErr) } -func doltCloneArgs(remoteURL, target string) []string { +// doltCloneArgs builds the `dolt clone` argv. A non-empty ref adds --ref so +// the clone reads a git-backed remote's Dolt data from that git ref instead +// of the default refs/dolt/data. +func doltCloneArgs(remoteURL, target, ref string) []string { args := []string{"clone"} if user := os.Getenv("DOLT_REMOTE_USER"); user != "" { args = append(args, "--user", user) } + if ref = strings.TrimSpace(ref); ref != "" { + args = append(args, "--ref", ref) + } return append(args, remoteURL, target) } diff --git a/internal/storage/dolt/bootstrap_test.go b/internal/storage/dolt/bootstrap_test.go index 4853457ac..642f2eead 100644 --- a/internal/storage/dolt/bootstrap_test.go +++ b/internal/storage/dolt/bootstrap_test.go @@ -19,7 +19,7 @@ func TestBootstrapCloneCmdEnvIsGuarded(t *testing.T) { t.Setenv("GIT_TRACE", "1") t.Setenv("GIT_CURL_VERBOSE", "1") - cmd := bootstrapCloneCmd(context.Background(), "git+https://example.com/repo.git", filepath.Join(t.TempDir(), "beads")) + cmd := bootstrapCloneCmd(context.Background(), "git+https://example.com/repo.git", filepath.Join(t.TempDir(), "beads"), "") if cmd.Env == nil { t.Fatal("bootstrapCloneCmd() left cmd.Env nil; the clone would inherit stderr-directed git tracing") } @@ -153,22 +153,70 @@ func TestBootstrapFromRemoteWithDB_PreservesPreExistingCloneTarget(t *testing.T) } } +// An invalid --ref is refused on a host whose database already exists, not +// only on a fresh one: the ref check runs before the already-exists early +// return, so the same argument is accepted or refused the same way everywhere. +// The valid-ref control proves the early return still fires, so the refusal +// comes from the ref and not from the call failing for some other reason. +// Needs no dolt binary — both outcomes are decided before the CLI lookup. +func TestBootstrapFromRemoteWithDBRef_RefusesInvalidRefOnBootstrappedHost(t *testing.T) { + doltDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(doltDir, "beads", ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + if !doltExists(doltDir) { + t.Fatal("test setup: doltExists should see the pre-existing database") + } + + for _, ref := range []string{"-x", "refs/heads/issue data"} { + cloned, err := BootstrapFromRemoteWithDBRef(context.Background(), doltDir, "git+https://example.com/repo.git", "beads", ref) + if err == nil || !strings.Contains(err.Error(), "invalid git data ref") { + t.Errorf("BootstrapFromRemoteWithDBRef(ref=%q) = %v, %v; want the invalid-ref error", ref, cloned, err) + } + } + + cloned, err := BootstrapFromRemoteWithDBRef(context.Background(), doltDir, "git+https://example.com/repo.git", "beads", "refs/dolt/units/team-12542") + if cloned || err != nil { + t.Fatalf("BootstrapFromRemoteWithDBRef with a valid ref on an existing database = %v, %v; want false, nil", cloned, err) + } +} + func TestDoltCloneArgs(t *testing.T) { t.Setenv("DOLT_REMOTE_USER", "") - got := doltCloneArgs("https://example.com/repo", "/tmp/clone") + got := doltCloneArgs("https://example.com/repo", "/tmp/clone", "") want := []string{"clone", "https://example.com/repo", "/tmp/clone"} if strings.Join(got, "\x00") != strings.Join(want, "\x00") { t.Fatalf("doltCloneArgs() = %q, want %q", got, want) } t.Setenv("DOLT_REMOTE_USER", "alice") - got = doltCloneArgs("https://example.com/repo", "/tmp/clone") + got = doltCloneArgs("https://example.com/repo", "/tmp/clone", "") want = []string{"clone", "--user", "alice", "https://example.com/repo", "/tmp/clone"} if strings.Join(got, "\x00") != strings.Join(want, "\x00") { t.Fatalf("doltCloneArgs() = %q, want %q", got, want) } } +// A git data ref reaches `dolt clone` as --ref, verbatim, whether it is a +// branch under refs/heads/ or a ref elsewhere in the namespace. +func TestDoltCloneArgsWithRef(t *testing.T) { + t.Setenv("DOLT_REMOTE_USER", "") + for _, ref := range []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} { + got := doltCloneArgs("git+https://example.com/repo.git", "/tmp/clone", ref) + want := []string{"clone", "--ref", ref, "git+https://example.com/repo.git", "/tmp/clone"} + if strings.Join(got, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("doltCloneArgs(ref=%q) = %q, want %q", ref, got, want) + } + } + + t.Setenv("DOLT_REMOTE_USER", "alice") + got := doltCloneArgs("git+https://example.com/repo.git", "/tmp/clone", " refs/heads/issue-data ") + want := []string{"clone", "--user", "alice", "--ref", "refs/heads/issue-data", "git+https://example.com/repo.git", "/tmp/clone"} + if strings.Join(got, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("doltCloneArgs() = %q, want %q", got, want) + } +} + func TestRemoveFailedCloneTargetWithRetryRemovesPartialClone(t *testing.T) { oldDelays := failedCloneCleanupRetryDelays failedCloneCleanupRetryDelays = []time.Duration{0} diff --git a/internal/storage/dolt/cli_mirror_ref_test.go b/internal/storage/dolt/cli_mirror_ref_test.go new file mode 100644 index 000000000..44d9defb3 --- /dev/null +++ b/internal/storage/dolt/cli_mirror_ref_test.go @@ -0,0 +1,267 @@ +package dolt + +import ( + "context" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" +) + +// writeCLIRepoState gives cliDir a .dolt/repo_state.json with one remote, +// the state a served database directory holds, without a working Dolt +// repository behind it: `dolt remote -v` there fails or lists nothing, which +// is also what the proxied CLI reports at a server's cold start. +func writeCLIRepoState(t *testing.T, cliDir, name, url, ref string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(cliDir, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + params := "{}" + if ref != "" { + params = `{"git_ref":"` + ref + `"}` + } + body := `{"head":"refs/heads/main","remotes":{"` + name + `":{"name":"` + name + `","url":"` + url + `","fetch_specs":["refs/heads/*:refs/remotes/` + name + `/*"],"params":` + params + `}},"backups":{},"branches":{}}` + if err := os.WriteFile(filepath.Join(cliDir, ".dolt", "repo_state.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func newServedStoreForCLIMirror(t *testing.T) (*DoltStore, string) { + t.Helper() + // These tests stand in for a proxied CLI that lists nothing; they must + // not depend on what a real dolt binary does in a directory that holds + // only a state file, so no dolt is reachable from PATH here. + t.Setenv("PATH", t.TempDir()) + root := t.TempDir() + store := &DoltStore{serverMode: true, dbPath: root, database: "beads"} + cliDir := store.CLIDir() + if cliDir != filepath.Join(root, "beads") { + t.Fatalf("CLIDir() = %q, want %q", cliDir, filepath.Join(root, "beads")) + } + return store, cliDir +} + +// When the CLI in cliDir lists nothing, the mirror check for a remote on a +// custom ref consults the state file in cliDir: matching URL and ref is a +// match and nothing is re-materialized; a different ref is not a match. A +// remote on the default ref keeps the previous behavior and does not consult +// the file. Both ref shapes. +func TestHasMatchingCLIRemoteFallsBackToPersistedStateForRef(t *testing.T) { + const url = "git+https://example.com/repo.git" + for _, ref := range []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} { + t.Run(ref, func(t *testing.T) { + store, cliDir := newServedStoreForCLIMirror(t) + writeCLIRepoState(t, cliDir, "origin", url, ref) + + if !store.hasMatchingCLIRemote("origin", url, ref) { + t.Fatal("same URL and ref recorded in cliDir should match") + } + if store.hasMatchingCLIRemote("origin", url, "refs/heads/elsewhere") { + t.Fatal("a different ref must not match") + } + if store.hasMatchingCLIRemote("origin", "git+https://example.com/other.git", ref) { + t.Fatal("a different URL must not match") + } + if err := store.ensureMatchingCLIRemote("origin", url, ref); err != nil { + t.Fatalf("ensureMatchingCLIRemote on a matching mirror should not touch dolt: %v", err) + } + }) + } + + t.Run("default ref does not consult the state file", func(t *testing.T) { + store, cliDir := newServedStoreForCLIMirror(t) + writeCLIRepoState(t, cliDir, "origin", url, "") + if store.hasMatchingCLIRemote("origin", url, "") { + t.Fatal("without a ref the check must stay on the CLI listing, which is empty here") + } + }) +} + +const listRemotesQuery = "SELECT name, url, params FROM dolt_remotes" + +// The git-protocol route decider carries the ref from the SQL listing into +// the mirror check: a served directory whose state file records the same +// URL and ref routes over the CLI; the same URL on another ref does not +// (the re-materialization then needs the dolt binary, absent here, and the +// error says so instead of a silent default). Dropping the ref at the call +// site would pass the first case on the default ref and fail this test. +func TestPrepareCLIRouteForGitProtocolCarriesRef(t *testing.T) { + const url = "git+ssh://git@example.com/org/repo.git" + for _, ref := range []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} { + t.Run(ref, func(t *testing.T) { + store, cliDir := newServedStoreForCLIMirror(t) + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + store.db = db + store.remote = "origin" + mock.MatchExpectationsInOrder(false) + // One listing per prepareCLIRouteForGitProtocol call below. + for i := 0; i < 2; i++ { + mock.ExpectQuery(regexp.QuoteMeta(listRemotesQuery)).WillReturnRows( + sqlmock.NewRows([]string{"name", "url", "params"}).AddRow("origin", url, `{"git_ref":"`+ref+`"}`)) + } + + writeCLIRepoState(t, cliDir, "origin", url, ref) + useCLI, err := store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err != nil || !useCLI { + t.Fatalf("matching state file: useCLI=%v err=%v, want true, nil", useCLI, err) + } + + writeCLIRepoState(t, cliDir, "origin", url, "refs/heads/elsewhere") + useCLI, err = store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err == nil || useCLI { + t.Fatalf("state file on another ref: useCLI=%v err=%v, want an error from the re-materialization", useCLI, err) + } + }) + } +} + +// In the GH#2118 cold-start window (SQL lists nothing, the state file has the +// remote) a remote on a custom ref is re-added over bd's SQL connection with +// --ref before the CLI route is taken, so the server the proxied transfer +// delegates to knows the remote; "already exists" from the server is the +// state wanted. That write happens only when this process owns the served +// directory (localActiveDatabaseDir is the CLI directory); for any other +// server the route fails with the remedy instead of registering a remote +// read from a client-local file. A remote on the default ref keeps the +// previous path and issues no SQL add. +func TestPrepareCLIRouteForGitProtocolColdStartReaddsRefRemoteOverSQL(t *testing.T) { + const url = "git+ssh://git@example.com/org/repo.git" + const ref = "refs/dolt/units/team-12542" + newStore := func(t *testing.T) (*DoltStore, string, sqlmock.Sqlmock) { + t.Helper() + store, cliDir := newServedStoreForCLIMirror(t) + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + store.db = db + store.remote = "origin" + store.localActiveDatabaseDir = cliDir + mock.MatchExpectationsInOrder(false) + // One listing per prepareCLIRouteForGitProtocol call. + mock.ExpectQuery(regexp.QuoteMeta(listRemotesQuery)).WillReturnRows(sqlmock.NewRows([]string{"name", "url", "params"})) + + return store, cliDir, mock + } + + t.Run("ref remote is re-added with --ref", func(t *testing.T) { + store, cliDir, mock := newStore(t) + writeCLIRepoState(t, cliDir, "origin", url, ref) + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE('add', '--ref', ?, ?, ?)")). + WithArgs(ref, "origin", url).WillReturnResult(sqlmock.NewResult(0, 1)) + useCLI, err := store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err != nil || !useCLI { + t.Fatalf("useCLI=%v err=%v, want true, nil", useCLI, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) + + t.Run("already exists is accepted", func(t *testing.T) { + store, cliDir, mock := newStore(t) + writeCLIRepoState(t, cliDir, "origin", url, ref) + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE('add', '--ref', ?, ?, ?)")). + WithArgs(ref, "origin", url).WillReturnError(errRemoteAlreadyExistsForTest) + useCLI, err := store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err != nil || !useCLI { + t.Fatalf("useCLI=%v err=%v, want true, nil", useCLI, err) + } + }) + + t.Run("unowned server is not written to", func(t *testing.T) { + store, cliDir, mock := newStore(t) + store.localActiveDatabaseDir = "" + writeCLIRepoState(t, cliDir, "origin", url, ref) + // No ExpectExec: an add would surface as a sqlmock error wrapped as + // the SQL re-add failure, which the assertion below rejects. + _, err := store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err == nil { + t.Fatal("an unowned server must not be written to; the route must fail") + } + // The remedy must be something that works at this commit and + // repairs the condition: the procedure call over bd's SQL + // connection to that server. + for _, want := range []string{"does not own", `bd sql 'CALL DOLT_REMOTE("add", "--ref", "` + ref + `", "origin", "` + url + `")'`} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should contain %q: %v", want, err) + } + } + if strings.Contains(err.Error(), "over SQL") { + t.Fatalf("no SQL add may be attempted for an unowned server: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) + + t.Run("other SQL failure is reported", func(t *testing.T) { + store, cliDir, mock := newStore(t) + writeCLIRepoState(t, cliDir, "origin", url, ref) + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE('add', '--ref', ?, ?, ?)")). + WithArgs(ref, "origin", url).WillReturnError(errSQLDownForTest) + _, err := store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err == nil || !strings.Contains(err.Error(), "GH#2118") || !strings.Contains(err.Error(), ref) { + t.Fatalf("err = %v, want the cold-start explanation naming the ref", err) + } + }) + + t.Run("default ref issues no SQL add", func(t *testing.T) { + store, cliDir, mock := newStore(t) + writeCLIRepoState(t, cliDir, "origin", url, "") + // No ExpectExec: an unexpected add comes back from sqlmock as an + // error that the code would wrap as the SQL re-add failure. The + // error seen must instead be the CLI re-materialization failing + // (no dolt repository behind the state file), the previous behavior. + _, err := store.prepareCLIRouteForGitProtocol(context.Background(), "origin") + if err == nil { + t.Fatal("without a usable CLI mirror the route must fail") + } + if strings.Contains(err.Error(), "over SQL") { + t.Fatalf("a default-ref remote must not be re-added over SQL: %v", err) + } + if !strings.Contains(err.Error(), "CLI") { + t.Fatalf("expected the CLI materialization failure, got: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) +} + +// The remedy is pasted into a shell, so user data in it must not change the +// command: an apostrophe in the URL is escaped for the shell, a dollar sign +// and a backtick sit inside single quotes, and a double quote or backslash in +// a value is escaped for the SQL literal. +func TestColdStartReaddRemedyQuotes(t *testing.T) { + got := coldStartReaddRemedy("origin", "git+https://example.com/o'hare/$HOME/`id`/repo.git", "refs/heads/issue-data") + want := `bd sql 'CALL DOLT_REMOTE("add", "--ref", "refs/heads/issue-data", "origin", "git+https://example.com/o'\''hare/$HOME/` + "`id`" + `/repo.git")'` + if got != want { + t.Fatalf("coldStartReaddRemedy =\n%s\nwant\n%s", got, want) + } + if !strings.HasPrefix(got, "bd sql '") || !strings.HasSuffix(got, "'") { + t.Fatalf("the statement must be one single-quoted shell word: %s", got) + } + got = coldStartReaddRemedy("origin", `git+https://example.com/a"b\c.git`, "refs/dolt/units/team-12542") + if !strings.Contains(got, `"git+https://example.com/a\"b\\c.git"`) { + t.Fatalf("a double quote and a backslash must be escaped inside the SQL literal: %s", got) + } +} + +var ( + errRemoteAlreadyExistsForTest = &testSQLError{"error: remote already exists"} + errSQLDownForTest = &testSQLError{"driver: bad connection"} +) + +type testSQLError struct{ msg string } + +func (e *testSQLError) Error() string { return e.msg } diff --git a/internal/storage/dolt/credentials.go b/internal/storage/dolt/credentials.go index 700e71c8a..64db95f08 100644 --- a/internal/storage/dolt/credentials.go +++ b/internal/storage/dolt/credentials.go @@ -637,7 +637,7 @@ func (s *DoltStore) prepareCLIRouteForPeerCredentials(ctx context.Context, peer } for _, r := range remotes { if r.Name == peer { - if err := s.ensureMatchingCLIRemote(peer, r.URL); err != nil { + if err := s.ensureMatchingCLIRemote(peer, r.URL, r.Ref); err != nil { return false, fmt.Errorf("peer remote %q has credentials and requires CLI routing: %w", peer, err) } return true, nil @@ -687,7 +687,7 @@ func (s *DoltStore) prepareCLIRouteForCredentials(ctx context.Context, remote st } for _, r := range remotes { if r.Name == remote { - if err := s.ensureMatchingCLIRemote(remote, r.URL); err != nil { + if err := s.ensureMatchingCLIRemote(remote, r.URL, r.Ref); err != nil { return false, fmt.Errorf("remote %q has credentials and requires CLI routing: %w", remote, err) } return true, nil @@ -713,7 +713,7 @@ func (s *DoltStore) shouldUseCLIForLocalRemoteWithError(ctx context.Context, rem } for _, r := range sqlRemotes { if r.Name == remote { - return s.hasMatchingCLIRemote(remote, r.URL), nil + return s.hasMatchingCLIRemote(remote, r.URL, r.Ref), nil } } return false, nil @@ -806,10 +806,11 @@ func (s *DoltStore) prepareCLIRouteForCloudAuth(ctx context.Context, remote stri if err != nil { return false, fmt.Errorf("list Dolt remotes before cloud-auth routing for remote %q: %w", remote, err) } - var remoteURL string + var remoteURL, remoteRef string for _, r := range remotes { if r.Name == remote { remoteURL = r.URL + remoteRef = r.Ref break } } @@ -821,7 +822,7 @@ func (s *DoltStore) prepareCLIRouteForCloudAuth(ctx context.Context, remote stri return false, nil // unknown scheme — not a cloud remote } if execenv.ContainsKeyWithPrefix(os.Environ(), prefixes...) { - if err := s.ensureMatchingCLIRemote(remote, remoteURL); err != nil { + if err := s.ensureMatchingCLIRemote(remote, remoteURL, remoteRef); err != nil { return false, fmt.Errorf("remote %q has cloud credentials and requires CLI routing: %w", remote, err) } return true, nil diff --git a/internal/storage/dolt/federation.go b/internal/storage/dolt/federation.go index cdc40cf31..2c8f01ce8 100644 --- a/internal/storage/dolt/federation.go +++ b/internal/storage/dolt/federation.go @@ -636,7 +636,7 @@ func (s *DoltStore) prepareCLIRouteForPeerGitProtocol(ctx context.Context, peer if !doltutil.IsGitProtocolURL(r.URL) { return false, nil } - if err := s.ensureMatchingCLIRemote(peer, r.URL); err != nil { + if err := s.ensureMatchingCLIRemote(peer, r.URL, r.Ref); err != nil { return false, fmt.Errorf("peer remote %q uses git protocol and requires CLI routing: %w", peer, err) } return true, nil diff --git a/internal/storage/dolt/git_remote_test.go b/internal/storage/dolt/git_remote_test.go index 29f06245d..5efc5e138 100644 --- a/internal/storage/dolt/git_remote_test.go +++ b/internal/storage/dolt/git_remote_test.go @@ -1646,7 +1646,7 @@ func TestCredentialCLIRoutingE2E(t *testing.T) { require.False(t, store.isGitProtocolRemote(ctx, store.remote), "file:// is not git-protocol") if !store.shouldUseCLIForCredentials(ctx, store.remote, store.mainRemoteCredentials()) { remotes, listErr := store.ListRemotes(ctx) - ensureErr := doltutil.EnsureCLIRemote(clientTestdbDir, store.remote, remoteURL) + ensureErr := doltutil.EnsureCLIRemote(clientTestdbDir, store.remote, remoteURL, "") t.Fatalf("should route through CLI for credentials; serverMode=%v remotes=%v listErr=%v cliRemote=%q ensureErr=%v", store.serverMode, remotes, listErr, doltutil.FindCLIRemote(clientTestdbDir, store.remote), ensureErr) } diff --git a/internal/storage/dolt/remote_routing_test.go b/internal/storage/dolt/remote_routing_test.go index 20468a4c9..5340b3a3a 100644 --- a/internal/storage/dolt/remote_routing_test.go +++ b/internal/storage/dolt/remote_routing_test.go @@ -19,7 +19,7 @@ func TestEnsureMatchingCLIRemoteSurfacesValidationErrors(t *testing.T) { database: "beads", } - err := store.ensureMatchingCLIRemote("origin", "ftp://server/path") + err := store.ensureMatchingCLIRemote("origin", "ftp://server/path", "") if err == nil { t.Fatal("expected invalid remote URL to be returned as an error") } @@ -201,8 +201,8 @@ func TestPrepareCLIRouteForGitProtocolColdStartWindow(t *testing.T) { // dolt_remotes. mock.MatchExpectationsInOrder(false) for i := 0; i < 4; i++ { - mock.ExpectQuery("SELECT name, url FROM dolt_remotes"). - WillReturnRows(sqlmock.NewRows([]string{"name", "url"})) + mock.ExpectQuery("SELECT name, url, params FROM dolt_remotes"). + WillReturnRows(sqlmock.NewRows([]string{"name", "url", "params"})) } store := &DoltStore{ serverMode: true, diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index ae80c6735..eae16e734 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -3977,9 +3977,11 @@ func (s *DoltStore) buildBatchCommitMessage(ctx context.Context, actor string) s } // hasMatchingCLIRemote reports whether the local CLI directory contains the -// same remote URL that SQL reports. CLI push/pull/fetch run from CLIDir, so -// SQL visibility alone is not enough to route safely. -func (s *DoltStore) hasMatchingCLIRemote(remote, expectedURL string) bool { +// same remote URL, on the same git data ref, that SQL reports. CLI +// push/pull/fetch run from CLIDir, so SQL visibility alone is not enough to +// route safely; a mirror on a different ref would move the data to the wrong +// ref of the git remote. An empty expectedRef is Dolt's default. +func (s *DoltStore) hasMatchingCLIRemote(remote, expectedURL, expectedRef string) bool { if expectedURL == "" { return false } @@ -3990,7 +3992,38 @@ func (s *DoltStore) hasMatchingCLIRemote(remote, expectedURL string) bool { if !s.hasCLIDatabase() { return false } - return doltutil.RemoteURLsMatch(doltutil.FindCLIRemote(cliDir, remote), expectedURL) + cliURL := doltutil.FindCLIRemote(cliDir, remote) + cliRef, err := doltutil.FindCLIRemoteRef(cliDir, remote) + if err != nil { + return false + } + if cliURL == "" && expectedRef != "" { + // A proxied dolt CLI can list nothing at cold start (GH#2118) and + // refuses remote parameters, so for a ref the server's own state + // file in cliDir is what says whether the mirror is in place. + if persisted, ok := persistedCLIRemote(cliDir, remote); ok { + cliURL, cliRef = persisted.URL, persisted.Ref + } + } + if !doltutil.RemoteURLsMatch(cliURL, expectedURL) { + return false + } + return storage.RemoteRefsMatch(cliRef, expectedRef) +} + +// persistedCLIRemote returns the named remote as recorded in cliDir's +// .dolt/repo_state.json. +func persistedCLIRemote(cliDir, remote string) (storage.RemoteInfo, bool) { + remotes, err := doltutil.PersistedRemotes(cliDir) + if err != nil { + return storage.RemoteInfo{}, false + } + for _, r := range remotes { + if r.Name == remote { + return r, true + } + } + return storage.RemoteInfo{}, false } // hasCLIDatabase reports whether CLIDir points at an initialized Dolt database. @@ -4008,8 +4041,8 @@ func (s *DoltStore) hasCLIDatabase() bool { // ensureMatchingCLIRemote materializes the local CLI remote needed before // subprocess push/pull/fetch routing. SQL remains the source of truth; the CLI // remote is only the local transport surface that dolt subprocesses read. -func (s *DoltStore) ensureMatchingCLIRemote(remote, expectedURL string) error { - if s.hasMatchingCLIRemote(remote, expectedURL) { +func (s *DoltStore) ensureMatchingCLIRemote(remote, expectedURL, expectedRef string) error { + if s.hasMatchingCLIRemote(remote, expectedURL, expectedRef) { return nil } cliDir := s.CLIDir() @@ -4019,11 +4052,11 @@ func (s *DoltStore) ensureMatchingCLIRemote(remote, expectedURL string) error { if cliDir == "" { return fmt.Errorf("remote %q (%s) requires CLI routing but no CLI directory is configured", remote, expectedURL) } - if err := doltutil.EnsureCLIRemote(cliDir, remote, expectedURL); err != nil { + if err := doltutil.EnsureCLIRemote(cliDir, remote, expectedURL, expectedRef); err != nil { return fmt.Errorf("materialize CLI remote %q (%s) in %s: %w", remote, expectedURL, cliDir, err) } - if !s.hasMatchingCLIRemote(remote, expectedURL) { - return fmt.Errorf("materialized CLI remote %q in %s, but its URL does not match SQL URL %q", remote, cliDir, expectedURL) + if !s.hasMatchingCLIRemote(remote, expectedURL, expectedRef) { + return fmt.Errorf("materialized CLI remote %q in %s, but its URL or git data ref does not match SQL (%s on %s)", remote, cliDir, expectedURL, storage.EffectiveGitDataRef(expectedRef)) } return nil } @@ -4075,7 +4108,7 @@ func (s *DoltStore) prepareCLIRouteForGitProtocol(ctx context.Context, remote st if !doltutil.IsGitProtocolURL(r.URL) { return false, nil } - if err := s.ensureMatchingCLIRemote(remote, r.URL); err != nil { + if err := s.ensureMatchingCLIRemote(remote, r.URL, r.Ref); err != nil { return false, fmt.Errorf("remote %q uses git protocol and requires CLI routing: %w", remote, err) } return true, nil @@ -4095,7 +4128,22 @@ func (s *DoltStore) prepareCLIRouteForGitProtocol(ctx context.Context, remote st if !doltutil.IsGitProtocolURL(r.URL) { return false, fmt.Errorf("remote %q (%s) is persisted on disk but not yet visible to this sql-server (GH#2118 cold start); retry shortly, or restart the dolt sql-server if it persists", remote, r.URL) } - if err := s.ensureMatchingCLIRemote(remote, r.URL); err != nil { + if r.Ref != "" { + // The proxied CLI refuses remote parameters, so a ref remote is + // given to the server over SQL, and only when this process owns + // the served directory: a client-local state file proves nothing + // about an external server. The remedy names the procedure the + // server accepts (the proxied CLI in its directory would refuse + // the same --ref, and bd's own remote add finds the persisted + // remote and returns without a SQL add). + if s.localActiveDatabaseDir == "" || s.localActiveDatabaseDir != s.CLIDir() { + return false, fmt.Errorf("remote %q (%s on %s) is persisted on disk but not visible to this sql-server (GH#2118 cold start), and this bd does not own the server's data directory; re-add it on that server with: %s", remote, r.URL, r.Ref, coldStartReaddRemedy(remote, r.URL, r.Ref)) + } + if err := versioncontrolops.AddRemote(ctx, s.db, remote, r.URL, r.Ref); err != nil && !strings.Contains(strings.ToLower(err.Error()), "already exists") { + return false, fmt.Errorf("remote %q (%s on %s) is persisted on disk but not visible to this sql-server (GH#2118 cold start), and re-adding it over SQL failed: %w", remote, r.URL, r.Ref, err) + } + } + if err := s.ensureMatchingCLIRemote(remote, r.URL, r.Ref); err != nil { return false, fmt.Errorf("remote %q uses git protocol and requires CLI routing: %w", remote, err) } return true, nil @@ -4103,6 +4151,19 @@ func (s *DoltStore) prepareCLIRouteForGitProtocol(ctx context.Context, remote st return false, nil } +// coldStartReaddRemedy is the command an operator runs to register a ref +// remote on a sql-server this bd does not own: bd sql with a DOLT_REMOTE call. +// The values are user data that lands in a message meant to be pasted into a +// shell, so the SQL literals are double-quoted with their escapes and the +// whole statement is single-quoted for the shell, which keeps an apostrophe, +// a dollar sign, or a backtick in a URL from changing the command. +func coldStartReaddRemedy(remote, url, ref string) string { + stmt := fmt.Sprintf("CALL DOLT_REMOTE(%s, %s, %s, %s, %s)", + doltutil.SQLDoubleQuoted("add"), doltutil.SQLDoubleQuoted("--ref"), + doltutil.SQLDoubleQuoted(ref), doltutil.SQLDoubleQuoted(remote), doltutil.SQLDoubleQuoted(url)) + return "bd sql " + doltutil.ShellQuote(stmt) +} + // shouldUseCLIForGitProtocol is a compatibility wrapper for tests and older // call sites. Prefer prepareCLIRouteForGitProtocol so mutation is explicit. func (s *DoltStore) shouldUseCLIForGitProtocol(ctx context.Context, remote string) (bool, error) { @@ -5500,8 +5561,13 @@ func (s *DoltStore) HasRemote(ctx context.Context, name string) (bool, error) { // AddRemote adds a Dolt remote func (s *DoltStore) AddRemote(ctx context.Context, name, url string) error { - _, err := s.db.ExecContext(ctx, "CALL DOLT_REMOTE('add', ?, ?)", name, url) - if err != nil { + return s.AddRemoteWithRef(ctx, name, url, "") +} + +// AddRemoteWithRef adds a remote whose Dolt data lives on the git ref ref; +// see storage.RemoteStore. +func (s *DoltStore) AddRemoteWithRef(ctx context.Context, name, url, ref string) error { + if err := versioncontrolops.AddRemote(ctx, s.db, name, url, ref); err != nil { return fmt.Errorf("failed to add remote %s: %w", name, err) } return nil diff --git a/internal/storage/doltutil/cli_mirror_ref_test.go b/internal/storage/doltutil/cli_mirror_ref_test.go new file mode 100644 index 000000000..ee9f87a9b --- /dev/null +++ b/internal/storage/doltutil/cli_mirror_ref_test.go @@ -0,0 +1,393 @@ +package doltutil + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" +) + +// initDoltRepo creates an empty Dolt repository in a temp directory with the +// dolt CLI. Skips when the CLI is absent. +func initDoltRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("dolt"); err != nil { + t.Skip("dolt CLI not available") + } + dir := t.TempDir() + cmd := exec.Command("dolt", "init", "--name", "bd-test", "--email", "bd-test@example.com") + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt init: %v\n%s", err, out) + } + return dir +} + +func readRepoState(t *testing.T, dir string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, ".dolt", "repo_state.json")) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// The CLI mirror that server-mode push, pull, and fetch shell out through +// must sit on the same git data ref as the SQL-visible remote. EnsureCLIRemote +// re-materializes the mirror when the ref differs, including a change to or +// from the default, and leaves the state file untouched when URL and ref +// already match. +func TestCLIMirrorFollowsGitDataRef(t *testing.T) { + for _, ref := range []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} { + t.Run(ref, func(t *testing.T) { + dir := initDoltRepo(t) + url := "git+https://example.com/repo.git" + + if err := EnsureCLIRemote(dir, "origin", url, ""); err != nil { + t.Fatalf("EnsureCLIRemote(default ref): %v", err) + } + if got, _ := FindCLIRemoteRef(dir, "origin"); got != "" { + t.Fatalf("default-ref mirror records Ref %q, want empty", got) + } + + if err := EnsureCLIRemote(dir, "origin", url, ref); err != nil { + t.Fatalf("EnsureCLIRemote(%s): %v", ref, err) + } + if got, _ := FindCLIRemoteRef(dir, "origin"); got != ref { + t.Fatalf("after ref change, mirror Ref = %q, want %q", got, ref) + } + if got := FindCLIRemote(dir, "origin"); !RemoteURLsMatch(got, url) { + t.Fatalf("after ref change, mirror URL = %q, want %q", got, url) + } + + before := readRepoState(t, dir) + if err := EnsureCLIRemote(dir, "origin", url, ref); err != nil { + t.Fatalf("EnsureCLIRemote(same URL and ref): %v", err) + } + if after := readRepoState(t, dir); after != before { + t.Errorf("matching URL and ref must leave repo_state.json untouched:\nbefore: %s\nafter: %s", before, after) + } + + if err := EnsureCLIRemote(dir, "origin", url, ""); err != nil { + t.Fatalf("EnsureCLIRemote(back to the default): %v", err) + } + if got, _ := FindCLIRemoteRef(dir, "origin"); got != "" { + t.Errorf("back on the default, mirror should record no git_ref, got %q", got) + } + if probes := probeRemotesLeft(t, dir); probes != 0 { + t.Errorf("%d probe remote(s) left behind", probes) + } + + if err := EnsureCLIRemote(dir, "origin", url, "refs/dolt/data"); err != nil { + t.Fatalf("EnsureCLIRemote(explicit default): %v", err) + } + // Dolt records the explicit value; it is the comparison that treats it + // as the default. + if got, _ := FindCLIRemoteRef(dir, "origin"); !storage.RemoteRefsMatch(got, "") { + t.Errorf("explicit default ref should compare equal to the default, got %q", got) + } + }) + } +} + +// Dolt refuses --ref for a remote that is not git-backed; the CLI mirror +// surfaces that refusal instead of adding the remote on the default ref. +func TestCLIMirrorRefRefusedForNonGitURL(t *testing.T) { + dir := initDoltRepo(t) + err := AddCLIRemoteWithRef(dir, "backup", "file://"+t.TempDir(), "refs/heads/issue-data") + if err == nil { + t.Fatal("AddCLIRemoteWithRef on file:// with a ref = nil, want error") + } + if !strings.Contains(err.Error(), "git remotes") { + t.Errorf("error should carry Dolt's --ref refusal, got: %v", err) + } + if got := FindCLIRemote(dir, "backup"); got != "" { + t.Errorf("refused remote must not exist, found %q", got) + } +} + +// probeRemotesLeft counts remotes named like EnsureCLIRemote's ref probe +// still recorded in dir; the probe must always be removed again. +func probeRemotesLeft(t *testing.T, dir string) int { + t.Helper() + remotes, err := PersistedRemotes(dir) + if err != nil { + t.Fatal(err) + } + n := 0 + for _, r := range remotes { + if strings.HasPrefix(r.Name, cliRefProbePrefix) { + n++ + } + } + return n +} + +// The probe runs before the real remote is removed, so a refused --ref (here +// dolt's own refusal on a file:// URL stands in for the proxied server's) +// leaves the remote exactly as it was and no probe behind. Reordering the +// probe after the removal, or dropping it, fails this test. +func TestCLIMirrorRefusedProbeLeavesRemoteInPlace(t *testing.T) { + dir := initDoltRepo(t) + url := "file://" + t.TempDir() + if err := AddCLIRemote(dir, "origin", url); err != nil { + t.Fatalf("AddCLIRemote: %v", err) + } + err := EnsureCLIRemote(dir, "origin", url, "refs/heads/issue-data") + if err == nil || !strings.Contains(err.Error(), "cannot record git data ref") { + t.Fatalf("EnsureCLIRemote with a ref on file:// = %v, want the probe's refusal", err) + } + if got := FindCLIRemote(dir, "origin"); !RemoteURLsMatch(got, url) { + t.Fatalf("origin must survive the refused probe, found %q", got) + } + if probes := probeRemotesLeft(t, dir); probes != 0 { + t.Errorf("%d probe remote(s) left behind", probes) + } +} + +// deadPID returns the pid of a process that has exited: the test binary run +// with a pattern no test matches. The kernel does not reuse a pid within the +// time this test takes. +func deadPID(t *testing.T) int { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^$") + if err := cmd.Run(); err != nil { + t.Fatalf("run a short-lived process: %v", err) + } + return cmd.Process.Pid +} + +// A probe left behind by a run killed between its add and its remove is +// swept before the next change to the mirror, so it does not stay in the +// state file (and in every remote listing) until removed by hand: this +// process's own leftover always, a dead run's where the platform can tell it +// from a live one (sweepsOtherRunsProbes). A remote that only shares the +// prefix, and the probe of a run still alive, are not touched. +func TestCLIMirrorSweepsLeftoverProbes(t *testing.T) { + dir := initDoltRepo(t) + url := "git+https://example.com/repo.git" + if err := AddCLIRemote(dir, "origin", url); err != nil { + t.Fatalf("AddCLIRemote: %v", err) + } + ownLeftover := fmt.Sprintf("%s%d", cliRefProbePrefix, os.Getpid()) + deadLeftover := fmt.Sprintf("%s%d", cliRefProbePrefix, deadPID(t)) + liveProbe := fmt.Sprintf("%s%d", cliRefProbePrefix, os.Getppid()) + userRemote := cliRefProbePrefix + "backup" + for _, name := range []string{ownLeftover, deadLeftover, liveProbe, userRemote} { + if err := AddCLIRemoteWithRef(dir, name, url, "refs/heads/issue-data"); err != nil { + t.Fatalf("plant %s: %v", name, err) + } + } + if err := EnsureCLIRemote(dir, "origin", url, "refs/dolt/units/team-12542"); err != nil { + t.Fatalf("EnsureCLIRemote: %v", err) + } + remotes, err := PersistedRemotes(dir) + if err != nil { + t.Fatal(err) + } + left := map[string]bool{} + for _, r := range remotes { + left[r.Name] = true + } + if left[ownLeftover] { + t.Errorf("%s should have been swept", ownLeftover) + } + if left[deadLeftover] == sweepsOtherRunsProbes { + t.Errorf("%s left=%v, want the opposite: a dead run's probe is swept only where the platform can tell it is dead", deadLeftover, left[deadLeftover]) + } + for _, kept := range []string{liveProbe, userRemote, "origin"} { + if !left[kept] { + t.Errorf("%s must not be swept", kept) + } + } + if got, _ := FindCLIRemoteRef(dir, "origin"); got != "refs/dolt/units/team-12542" { + t.Errorf("origin Ref = %q after re-materialization", got) + } + + // The sweep runs before any change to the mirror, a move back to the + // default ref included, and not only ahead of a probe. The own name is + // used so the case holds on every platform. + if err := AddCLIRemoteWithRef(dir, ownLeftover, url, "refs/heads/issue-data"); err != nil { + t.Fatalf("plant %s again: %v", ownLeftover, err) + } + if err := EnsureCLIRemote(dir, "origin", url, ""); err != nil { + t.Fatalf("EnsureCLIRemote(back to the default): %v", err) + } + remotes, err = PersistedRemotes(dir) + if err != nil { + t.Fatal(err) + } + for _, r := range remotes { + if r.Name == ownLeftover { + t.Errorf("%s should have been swept by the move to the default ref", ownLeftover) + } + } +} + +// An invalid ref is refused by EnsureCLIRemote before the mirror is touched +// even when a remote exists, the case where the check used to run inside the +// probe and come back worded as the server refusing parameters; the existing +// remote is left as it was. +func TestCLIMirrorInvalidRefIsRefusedBeforeProbe(t *testing.T) { + dir := initDoltRepo(t) + url := "git+https://example.com/repo.git" + if err := AddCLIRemote(dir, "origin", url); err != nil { + t.Fatalf("AddCLIRemote: %v", err) + } + err := EnsureCLIRemote(dir, "origin", url, "refs/heads/issue data") + if err == nil || !strings.Contains(err.Error(), "invalid git data ref") || strings.Contains(err.Error(), "refuses remote parameters") { + t.Errorf("EnsureCLIRemote with an invalid ref over an existing remote = %v, want the argument check's own error", err) + } + if got := FindCLIRemote(dir, "origin"); got != url { + t.Errorf("origin URL after the refusal = %q, want %q untouched", got, url) + } + if got, err := FindCLIRemoteRef(dir, "origin"); err != nil || got != "" { + t.Errorf("origin Ref after the refusal = %q, %v, want the default ref untouched", got, err) + } +} + +// leftoverProbe admits this process's own name and, where the platform can +// tell (sweepsOtherRunsProbes), the exact probe shape for a pid that is not +// running; a prefix alone, a padded or signed number, and a live pid are not +// leftovers. Needs no dolt binary. +func TestLeftoverProbe(t *testing.T) { + own := fmt.Sprintf("%s%d", cliRefProbePrefix, os.Getpid()) + dead := deadPID(t) + if !leftoverProbe(own, own) { + t.Errorf("leftoverProbe(%q) = false, want true", own) + } + if deadName := fmt.Sprintf("%s%d", cliRefProbePrefix, dead); leftoverProbe(deadName, own) != sweepsOtherRunsProbes { + t.Errorf("leftoverProbe(%q) = %v, want %v on this platform", deadName, !sweepsOtherRunsProbes, sweepsOtherRunsProbes) + } + for _, name := range []string{ + cliRefProbePrefix + "backup", cliRefProbePrefix, cliRefProbePrefix + "0", cliRefProbePrefix + "-5", + fmt.Sprintf("%s0%d", cliRefProbePrefix, dead), fmt.Sprintf("%s%d ", cliRefProbePrefix, dead), + fmt.Sprintf("%s%d", cliRefProbePrefix, os.Getppid()), "origin", + } { + if leftoverProbe(name, own) { + t.Errorf("leftoverProbe(%q) = true, want false", name) + } + } +} + +// When the state file cannot be read at sweep time, the sweep reports that +// read, not a refusal of remote parameters by the server. The sweep is called +// directly because through EnsureCLIRemote the same file is read a moment +// earlier by FindCLIRemoteRef, which refuses first. Needs no dolt binary: the +// error is returned before any dolt invocation. +func TestCLIMirrorSweepReportsUnreadableState(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".dolt", "repo_state.json"), []byte(`{"remotes": not-json`), 0o644); err != nil { + t.Fatal(err) + } + err := sweepLeftoverProbes(dir) + if err == nil { + t.Fatal("sweepLeftoverProbes on a corrupt state file = nil, want the read error") + } + if !strings.Contains(err.Error(), "recorded remotes") || strings.Contains(err.Error(), "refuses remote parameters") { + t.Fatalf("sweepLeftoverProbes error must name the state-file read, not the server's refusal: %v", err) + } +} + +// A leftover under this process's own name that cannot be removed is reported +// as that removal, never as the server refusing parameters: here the state +// file names it but no dolt repository backs it, so the removal fails. +// Reached through EnsureCLIRemote: the failed CLI listing is discarded, the +// state file is read and shows no origin, and the sweep runs before the add. +// Needs no dolt binary (without one, the removal fails on the missing +// binary). +func TestCLIMirrorOwnLeftoverRemovalFailureIsReported(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + own := ownCLIRefProbe() + state := `{"head":"refs/heads/main","remotes":{"` + own + `":{"name":"` + own + `","url":"git+https://example.com/repo.git","fetch_specs":[],"params":{"git_ref":"refs/heads/issue-data"}}},"backups":{},"branches":{}}` + if err := os.WriteFile(filepath.Join(dir, ".dolt", "repo_state.json"), []byte(state), 0o644); err != nil { + t.Fatal(err) + } + err := EnsureCLIRemote(dir, "origin", "git+https://example.com/repo.git", "refs/heads/issue-data") + if err == nil || !strings.Contains(err.Error(), "remove leftover ref probe remote "+own) || strings.Contains(err.Error(), "refuses remote parameters") { + t.Fatalf("EnsureCLIRemote = %v, want the own-leftover removal failure", err) + } +} + +// An invalid ref is refused by EnsureCLIRemote itself, before the state file +// is read and before any dolt invocation, and not under the probe's wording +// about a server refusing parameters. The state file is made unreadable so +// that only the check at the top of EnsureCLIRemote can produce this error: +// without it, the read of the recorded ref fails first. Needs no dolt binary. +func TestCLIMirrorInvalidRefIsRefusedUnwrapped(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".dolt", "repo_state.json"), []byte(`{"remotes": not-json`), 0o644); err != nil { + t.Fatal(err) + } + err := EnsureCLIRemote(dir, "origin", "git+https://example.com/repo.git", "refs/heads/issue data") + if err == nil || !strings.Contains(err.Error(), "invalid git data ref") || strings.Contains(err.Error(), "recorded ref") || strings.Contains(err.Error(), "refuses remote parameters") { + t.Fatalf("EnsureCLIRemote with an invalid ref = %v, want the argument check's own error before the state file is read", err) + } +} + +// The argument check refuses the bytes git's ref rules refuse, a space, an +// ASCII control character, DEL, plus a leading dash so the value cannot be +// mistaken for an option; dolt accepts all of them through exec, so this is +// bd policy at the argv boundary. A non-ASCII space (U+00A0, U+2003, U+0085) +// is a valid ref byte for git check-ref-format and for dolt remote add +// --ref, and passes: this layer adds no ref-name rules of its own. Both ref +// shapes and the empty default pass. +func TestValidateGitDataRefArg(t *testing.T) { + for _, ok := range []string{ + "", "refs/heads/issue-data", "refs/dolt/units/team-12542", "refs/dolt/data", + "refs/heads/issue\u00a0data", "refs/heads/issue\u2003data", "refs/heads/issue\u0085data", + } { + if err := ValidateGitDataRefArg(ok); err != nil { + t.Errorf("ValidateGitDataRefArg(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{ + "-refs/heads/x", "--ref", "refs/heads/issue data", "refs/heads/issue\tdata", "refs/heads/issue\ndata", + "refs/heads/issue\x7fdata", "refs/heads/issue\x1bdata", "refs/heads/issue\x00data", + } { + if err := ValidateGitDataRefArg(bad); err == nil { + t.Errorf("ValidateGitDataRefArg(%q) = nil, want an error", bad) + } + } +} + +// A state file that cannot be parsed is an error out of EnsureCLIRemote, +// before any dolt mutation: treating it as the default ref would remove a +// ref remote and re-add it on refs/dolt/data. Needs no dolt binary. +// +// The empty-ref case is asserted too, because that is the one a later +// "helpful" relaxation would gate the read error on (only escalate when a ref +// was asked for). It must stay an error: an unreadable file cannot rule out a +// ref on the mirror, so the default-ref route has no more proof that the +// mirror is already correct than the ref route does. +func TestCLIMirrorUnreadableStateIsAnError(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".dolt", "repo_state.json"), []byte(`{"remotes": not-json`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := FindCLIRemoteRef(dir, "origin"); err == nil { + t.Fatal("FindCLIRemoteRef on a corrupt state file = nil error, want error") + } + for _, ref := range []string{"refs/dolt/units/team-12542", ""} { + err := EnsureCLIRemote(dir, "origin", "git+https://example.com/repo.git", ref) + if err == nil || !strings.Contains(err.Error(), "recorded ref") { + t.Fatalf("EnsureCLIRemote(ref=%q) on a corrupt state file = %v, want the read error", ref, err) + } + } +} diff --git a/internal/storage/doltutil/persisted_remotes_test.go b/internal/storage/doltutil/persisted_remotes_test.go index 40ae7db82..bd632e86a 100644 --- a/internal/storage/doltutil/persisted_remotes_test.go +++ b/internal/storage/doltutil/persisted_remotes_test.go @@ -3,6 +3,7 @@ package doltutil import ( "os" "path/filepath" + "strings" "testing" ) @@ -54,10 +55,59 @@ func TestPersistedRemotes(t *testing.T) { } }) + t.Run("git_ref parameter is surfaced as Ref", func(t *testing.T) { + dir := writeState(t, `{"remotes":{ + "branch":{"name":"branch","url":"git+https://example.com/repo.git","fetch_specs":["refs/heads/*:refs/remotes/branch/*"],"params":{"git_ref":"refs/heads/issue-data"}}, + "plain":{"name":"plain","url":"file:///tmp/p","fetch_specs":["refs/heads/*:refs/remotes/plain/*"],"params":{}}, + "unit":{"name":"unit","url":"git+file:///srv/ledgers","fetch_specs":["refs/heads/*:refs/remotes/unit/*"],"params":{"git_ref":"refs/dolt/units/team-12542"}} + }}`) + remotes, err := PersistedRemotes(dir) + if err != nil { + t.Fatalf("PersistedRemotes: %v", err) + } + want := map[string]string{"branch": "refs/heads/issue-data", "plain": "", "unit": "refs/dolt/units/team-12542"} + if len(remotes) != len(want) { + t.Fatalf("PersistedRemotes = %+v, want %d remotes", remotes, len(want)) + } + for _, r := range remotes { + if r.Ref != want[r.Name] { + t.Errorf("remote %s: Ref = %q, want %q", r.Name, r.Ref, want[r.Name]) + } + } + if got, _ := FindCLIRemoteRef(dir, "unit"); got != "refs/dolt/units/team-12542" { + t.Errorf("FindCLIRemoteRef(unit) = %q", got) + } + if got, _ := FindCLIRemoteRef(dir, "plain"); got != "" { + t.Errorf("FindCLIRemoteRef(plain) = %q, want empty", got) + } + if got, _ := FindCLIRemoteRef(dir, "absent"); got != "" { + t.Errorf("FindCLIRemoteRef(absent) = %q, want empty", got) + } + }) + t.Run("corrupt state file is an error, not silently none", func(t *testing.T) { dir := writeState(t, `{"remotes": not-json`) if _, err := PersistedRemotes(dir); err == nil { t.Fatal("PersistedRemotes on corrupt repo_state.json = nil error, want error") } }) + + // A git_ref of the wrong JSON type is "could not tell" too, not "no ref": + // silently reading it as the default is what would re-add a remote that is + // pinned to a custom ref onto refs/dolt/data. + t.Run("a git_ref that is not a string is an error", func(t *testing.T) { + dir := writeState(t, `{"remotes":{ + "unit":{"name":"unit","url":"git+file:///srv/ledgers","fetch_specs":[],"params":{"git_ref":["refs/dolt/units/team-12542"]}} + }}`) + remotes, err := PersistedRemotes(dir) + if err == nil { + t.Fatalf("PersistedRemotes with a non-string git_ref = %+v, nil; want an error", remotes) + } + if !strings.Contains(err.Error(), "unit") { + t.Errorf("error %q does not name the remote", err) + } + if _, err := FindCLIRemoteRef(dir, "unit"); err == nil { + t.Error("FindCLIRemoteRef with a non-string git_ref = nil error, want the read error") + } + }) } diff --git a/internal/storage/doltutil/probe_pid_other.go b/internal/storage/doltutil/probe_pid_other.go new file mode 100644 index 000000000..b7ac857ac --- /dev/null +++ b/internal/storage/doltutil/probe_pid_other.go @@ -0,0 +1,13 @@ +//go:build !unix && !windows + +package doltutil + +// sweepsOtherRunsProbes is false here: with no liveness check implemented, +// the sweep removes only this process's own leftover. +const sweepsOtherRunsProbes = false + +// probeProcessAlive has no liveness check implemented for this platform, so +// every other pid is treated as alive: the sweep then removes only this +// process's own leftover, never another run's, which errs on the side of +// leaving a remote in place. +func probeProcessAlive(_ int) bool { return true } diff --git a/internal/storage/doltutil/probe_pid_unix.go b/internal/storage/doltutil/probe_pid_unix.go new file mode 100644 index 000000000..5ce26fa11 --- /dev/null +++ b/internal/storage/doltutil/probe_pid_unix.go @@ -0,0 +1,28 @@ +//go:build unix + +package doltutil + +import ( + "errors" + "syscall" +) + +// sweepsOtherRunsProbes says the sweep can tell a dead run's probe from a +// live one on this platform, so it removes another run's leftover. +const sweepsOtherRunsProbes = true + +// probeProcessAlive reports whether a process with pid is running, with the +// kill(pid, 0) that internal/doltserver uses, read more strictly: there any +// error means dead, here only ESRCH ("no such process") does. EPERM means the +// process exists under another user, and any other errno means the check +// itself failed, and both report alive so the caller leaves the remote in +// place on an inconclusive read. kill(0, sig) would signal the caller's own +// process group and kill(-1, sig) everything it may signal, so a pid that is +// not positive never reaches the syscall and reports alive. +func probeProcessAlive(pid int) bool { + if pid <= 0 { + return true + } + err := syscall.Kill(pid, 0) + return !errors.Is(err, syscall.ESRCH) +} diff --git a/internal/storage/doltutil/probe_pid_windows.go b/internal/storage/doltutil/probe_pid_windows.go new file mode 100644 index 000000000..b142637fa --- /dev/null +++ b/internal/storage/doltutil/probe_pid_windows.go @@ -0,0 +1,37 @@ +//go:build windows + +package doltutil + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +// sweepsOtherRunsProbes says the sweep can tell a dead run's probe from a +// live one on this platform, so it removes another run's leftover. +const sweepsOtherRunsProbes = true + +// probeProcessAlive reports whether a process with pid is running, the way +// internal/doltserver does on Windows: opening the process is not enough, +// since an exited process stays openable while another handle refers to it, +// so a zero-timeout wait tells a signaled (exited) process from a running +// one. A pid that cannot be opened reads as dead, except for access denied, +// which means the process exists under another user and reads as alive; a +// wait that fails reads as alive too, so the caller leaves the remote in +// place on an inconclusive read. +func probeProcessAlive(pid int) bool { + if pid <= 0 { + return true + } + h, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(pid)) + if err != nil { + return errors.Is(err, windows.ERROR_ACCESS_DENIED) + } + defer func() { _ = windows.CloseHandle(h) }() + status, err := windows.WaitForSingleObject(h, 0) + if err != nil { + return true + } + return status != uint32(windows.WAIT_OBJECT_0) +} diff --git a/internal/storage/doltutil/remotes.go b/internal/storage/doltutil/remotes.go index 9702fd41e..b814a9e83 100644 --- a/internal/storage/doltutil/remotes.go +++ b/internal/storage/doltutil/remotes.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "sort" + "strconv" "strings" "sync" "time" @@ -85,7 +86,10 @@ func IsGitProtocolURL(url string) bool { // distinguishable (bd-6dnrw.33). A missing .dolt directory or repo_state.json // means "not a dolt repository here" and returns (nil, nil); an unreadable or // unparseable file returns an error so callers can tell "definitely none" -// from "could not tell". Results are sorted by name. +// from "could not tell". Results are sorted by name. Ref is the remote's +// git_ref parameter when recorded, else empty; a git_ref that is present but +// is not a string is unparseable in the same sense (see +// storage.GitRefFromParams) and is reported as an error naming the remote. func PersistedRemotes(dbPath string) ([]storage.RemoteInfo, error) { path := filepath.Join(dbPath, ".dolt", "repo_state.json") data, err := os.ReadFile(path) // #nosec G304 -- repo-local dolt state file @@ -97,7 +101,8 @@ func PersistedRemotes(dbPath string) ([]storage.RemoteInfo, error) { } var state struct { Remotes map[string]struct { - URL string `json:"url"` + URL string `json:"url"` + Params map[string]any `json:"params"` } `json:"remotes"` } if err := json.Unmarshal(data, &state); err != nil { @@ -105,7 +110,15 @@ func PersistedRemotes(dbPath string) ([]storage.RemoteInfo, error) { } remotes := make([]storage.RemoteInfo, 0, len(state.Remotes)) for name, r := range state.Remotes { - remotes = append(remotes, storage.RemoteInfo{Name: name, URL: r.URL}) + ref, err := storage.GitRefFromParams(r.Params) + if err != nil { + return nil, fmt.Errorf("remote %s in %s: %w", name, path, err) + } + remotes = append(remotes, storage.RemoteInfo{ + Name: name, + URL: r.URL, + Ref: ref, + }) } sort.Slice(remotes, func(i, j int) bool { return remotes[i].Name < remotes[j].Name }) return remotes, nil @@ -114,6 +127,8 @@ func PersistedRemotes(dbPath string) ([]storage.RemoteInfo, error) { // ListCLIRemotes parses `dolt remote -v` output from the given database // directory. This is a read-only guard for deciding whether CLI push/pull/fetch // can safely run from that directory; remote mutation still goes through SQL. +// Ref is not populated here (the params column is not parsed from the +// listing); callers that need it use FindCLIRemoteRef. func ListCLIRemotes(dbPath string) ([]storage.RemoteInfo, error) { ctx, cancel := context.WithTimeout(context.Background(), listCLIRemotesTimeout(dbPath)) defer cancel() @@ -151,17 +166,71 @@ func RemoteURLsMatch(got, want string) bool { return false } +// ValidateGitDataRefArg checks a git data ref before it becomes a dolt +// command-line argument. It refuses the bytes git's ref rules refuse, a +// space, an ASCII control character, and DEL, plus a leading dash, so the +// value can never be mistaken for an option. Dolt itself accepts all of them +// through exec, so this is bd policy at the argv boundary, not a ref-name +// check: a non-ASCII space such as U+00A0 is a valid ref byte for git and +// for dolt and passes here like any other byte, and the full ref-name rules +// belong to the command layer. An empty ref is valid and means Dolt's +// default. +func ValidateGitDataRefArg(ref string) error { + if ref == "" { + return nil + } + if strings.HasPrefix(ref, "-") { + return fmt.Errorf("git data ref %q must not start with a dash", ref) + } + for _, r := range ref { + if r < 0x20 || r == 0x7f || r == ' ' { + return fmt.Errorf("git data ref %q contains a space or an ASCII control character", ref) + } + } + return nil +} + +// SQLDoubleQuoted returns s as a MySQL string literal in double quotes, with +// the backslash and the double quote escaped, for a statement that is shown +// to a person to run rather than bound as a parameter. +func SQLDoubleQuoted(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `"`, `\"`) + return `"` + s + `"` +} + // AddCLIRemote adds a remote at the filesystem level via dolt CLI. // Remote mutation should normally go through SQL; this is reserved for the // local CLI mirror required by subprocess push/pull/fetch routing. +// +// It has no production caller: EnsureCLIRemote, its only one, now goes through +// AddCLIRemoteWithRef. It is kept deliberately as the default-ref spelling for +// tests and callers that have no ref to carry, so a reader changing +// AddCLIRemoteWithRef does not have to re-derive that. func AddCLIRemote(dbPath, name, url string) error { + return AddCLIRemoteWithRef(dbPath, name, url, "") +} + +// AddCLIRemoteWithRef is AddCLIRemote for a git-backed remote whose Dolt data +// lives on the git ref ref (`dolt remote add --ref`). An empty ref is +// AddCLIRemote. +func AddCLIRemoteWithRef(dbPath, name, url, ref string) error { if err := remotecache.ValidateRemoteName(name); err != nil { return fmt.Errorf("invalid remote name: %w", err) } if err := remotecache.ValidateRemoteURL(url); err != nil { return fmt.Errorf("invalid remote URL: %w", err) } - cmd := exec.Command("dolt", "remote", "add", name, url) // #nosec G204 -- validated argv + ref = strings.TrimSpace(ref) + if err := ValidateGitDataRefArg(ref); err != nil { + return fmt.Errorf("invalid git data ref: %w", err) + } + args := []string{"remote", "add"} + if ref != "" { + args = append(args, "--ref", ref) + } + args = append(args, name, url) + cmd := exec.Command("dolt", args...) // #nosec G204 -- validated argv cmd.Dir = dbPath out, err := cmd.CombinedOutput() if err != nil { @@ -199,35 +268,163 @@ func FindCLIRemote(dbPath, name string) string { return "" } -// EnsureCLIRemote makes the local CLI remote match the SQL-visible remote URL. -// It is intentionally idempotent and only mutates the CLI surface when the -// remote is absent or points somewhere else. -func EnsureCLIRemote(dbPath, name, url string) error { +// cliRefProbePrefix names the throwaway remote probeCLIRemoteRef adds. +const cliRefProbePrefix = "bd-ref-probe-" + +// ownCLIRefProbe names the throwaway remote this process's probe uses. +func ownCLIRefProbe() string { + return fmt.Sprintf("%s%d", cliRefProbePrefix, os.Getpid()) +} + +// sweepLeftoverProbes removes probe remotes left behind by a run killed +// between its add and its remove: a remote named exactly as probeCLIRemoteRef +// names its probes whose pid is no longer running, and one under this +// process's own name (see leftoverProbe). EnsureCLIRemote runs it before +// every change to the mirror, whatever ref the change is onto, so a dead +// run's leftover is removed at the next re-materialization of that mirror +// where the platform has a liveness check (unix and windows; elsewhere only +// the own name is swept). For another run's leftover that removal is +// best-effort: a failure is warned about and tried again at the next change +// rather than failing the caller's own mirror change, and a pid an unrelated +// process has since taken keeps the leftover until that pid exits. A probe of +// a live run is left alone, so two bd processes re-materializing the same +// mirror at the same moment do not remove each other's probe. A leftover +// under this process's own name must go, or the probe's add would fail and be +// read as the server's refusal; its removal failing is reported as such. +func sweepLeftoverProbes(dbPath string) error { + own := ownCLIRefProbe() + remotes, err := PersistedRemotes(dbPath) + if err != nil { + return fmt.Errorf("read the recorded remotes in %s before changing the CLI mirror: %w", dbPath, err) + } + for _, r := range remotes { + if !leftoverProbe(r.Name, own) { + continue + } + if err := RemoveCLIRemote(dbPath, r.Name); err != nil { + if r.Name == own { + return fmt.Errorf("remove leftover ref probe remote %s in %s: %w", own, dbPath, err) + } + // Best-effort, but never silent: an un-removable leftover stays + // visible in `bd remote list` until some run succeeds, and without + // this line nothing ever says why. + fmt.Fprintf(os.Stderr, + "Warning: could not remove the ref probe remote %s left in %s by an exited run (will retry at the next mirror change): %v\n", + r.Name, dbPath, err) + } + } + return nil +} + +// probeCLIRemoteRef adds and removes a throwaway remote with url and ref +// before EnsureCLIRemote removes the real one: a dolt CLI proxied to a +// running sql-server refuses remote parameters, and that refusal must come +// before anything is deleted. The caller has swept leftovers already. +func probeCLIRemoteRef(dbPath, url, ref string) error { + if strings.TrimSpace(ref) == "" { + return nil + } + probe := ownCLIRefProbe() + if err := AddCLIRemoteWithRef(dbPath, probe, url, ref); err != nil { + return fmt.Errorf("cannot record git data ref %s on the CLI mirror in %s (a dolt sql-server serving this directory refuses remote parameters over the CLI): %w", ref, dbPath, err) + } + if err := RemoveCLIRemote(dbPath, probe); err != nil { + return fmt.Errorf("remove ref probe remote %s in %s: %w", probe, dbPath, err) + } + return nil +} + +// leftoverProbe reports whether the remote called name may be swept as a +// probe left by an earlier run: this process's own probe name, or the exact +// shape probeCLIRemoteRef generates (the prefix followed by a pid in +// canonical decimal, nothing else) for a pid that is not running. A remote +// that merely shares the prefix, such as one a user named bd-ref-probe-backup, +// and the probe of a run still alive are not leftovers. +func leftoverProbe(name, own string) bool { + if name == own { + return true + } + digits, ok := strings.CutPrefix(name, cliRefProbePrefix) + if !ok || digits == "" { + return false + } + pid, err := strconv.Atoi(digits) + if err != nil || pid <= 0 || strconv.Itoa(pid) != digits { + return false + } + return !probeProcessAlive(pid) +} + +// FindCLIRemoteRef returns the git data ref recorded for the named remote in +// dbPath's .dolt/repo_state.json, verbatim (compare with +// storage.RemoteRefsMatch), or "" when the remote is absent or carries no +// ref. A state file that cannot be read or parsed is an error, never the +// default ref: treating it as the default would feed a remove-and-re-add of +// a ref remote onto refs/dolt/data. +func FindCLIRemoteRef(dbPath, name string) (string, error) { + remotes, err := PersistedRemotes(dbPath) + if err != nil { + return "", err + } + for _, r := range remotes { + if r.Name == name { + return r.Ref, nil + } + } + return "", nil +} + +// EnsureCLIRemote makes the local CLI remote match the SQL-visible remote URL +// and git data ref (an empty ref is Dolt's default). It is intentionally +// idempotent and only mutates the CLI surface when the remote is absent, +// points somewhere else, or sits on a different ref; before it mutates, it +// sweeps probe remotes left by interrupted runs. +// +// Reading the mirror's recorded ref comes before the no-op short-circuit, so a +// state file that cannot be read fails the route for a default-ref remote too, +// not only for a remote with a ref. That is deliberate: an unreadable file +// cannot rule out a ref on the mirror, so a URL match alone does not prove the +// mirror is already correct, and continuing would push onto whatever ref the +// mirror is secretly pinned to. Failing is loud and mutates nothing. +func EnsureCLIRemote(dbPath, name, url, ref string) error { if err := remotecache.ValidateRemoteName(name); err != nil { return fmt.Errorf("invalid remote name: %w", err) } if err := remotecache.ValidateRemoteURL(url); err != nil { return fmt.Errorf("invalid remote URL: %w", err) } + if err := ValidateGitDataRefArg(strings.TrimSpace(ref)); err != nil { + return fmt.Errorf("invalid git data ref: %w", err) + } lock := cliRemoteLock(dbPath) lock.Lock() defer lock.Unlock() current := FindCLIRemote(dbPath, name) - if RemoteURLsMatch(current, url) { + currentRef, err := FindCLIRemoteRef(dbPath, name) + if err != nil { + return fmt.Errorf("read the recorded ref of CLI remote %q in %s: %w", name, dbPath, err) + } + if RemoteURLsMatch(current, url) && storage.RemoteRefsMatch(currentRef, ref) { return nil } + if err := sweepLeftoverProbes(dbPath); err != nil { + return err + } if current != "" { + if err := probeCLIRemoteRef(dbPath, url, ref); err != nil { + return err + } if err := RemoveCLIRemote(dbPath, name); err != nil { return err } } - if err := AddCLIRemote(dbPath, name, url); err != nil { + if err := AddCLIRemoteWithRef(dbPath, name, url, ref); err != nil { if current == "" { return err } - if restoreErr := AddCLIRemote(dbPath, name, current); restoreErr != nil { + if restoreErr := AddCLIRemoteWithRef(dbPath, name, current, currentRef); restoreErr != nil { return fmt.Errorf("add replacement CLI remote failed: %w; additionally failed to restore previous URL %q: %v", err, current, restoreErr) } return fmt.Errorf("add replacement CLI remote failed; previous URL %q restored: %w", current, err) diff --git a/internal/storage/domain/db/remote.go b/internal/storage/domain/db/remote.go index 0606f4ca1..69f9344ef 100644 --- a/internal/storage/domain/db/remote.go +++ b/internal/storage/domain/db/remote.go @@ -2,8 +2,11 @@ package db import ( "context" + "database/sql" "fmt" + "strings" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" ) @@ -28,6 +31,19 @@ func (r *remoteSQLRepositoryImpl) AddRemote(ctx context.Context, name, url strin return nil } +// AddRemoteWithRef passes a non-empty ref as DOLT_REMOTE's --ref, which Dolt +// records as the remote's git_ref parameter and accepts for git-backed +// remotes only. +func (r *remoteSQLRepositoryImpl) AddRemoteWithRef(ctx context.Context, name, url, ref string) error { + if ref = strings.TrimSpace(ref); ref == "" { + return r.AddRemote(ctx, name, url) + } + if err := r.vc.Remote(ctx, "add", "--ref", ref, name, url); err != nil { + return fmt.Errorf("db: AddRemoteWithRef %s: %w", name, err) + } + return nil +} + func (r *remoteSQLRepositoryImpl) RemoveRemote(ctx context.Context, name string) error { if err := r.vc.Remote(ctx, "remove", name); err != nil { return fmt.Errorf("db: RemoveRemote %s: %w", name, err) @@ -36,7 +52,7 @@ func (r *remoteSQLRepositoryImpl) RemoveRemote(ctx context.Context, name string) } func (r *remoteSQLRepositoryImpl) ListRemotes(ctx context.Context) ([]domain.Remote, error) { - rows, err := r.runner.QueryContext(ctx, "SELECT name, url FROM dolt_remotes") + rows, err := r.runner.QueryContext(ctx, "SELECT name, url, params FROM dolt_remotes") if err != nil { return nil, fmt.Errorf("db: ListRemotes: query: %w", err) } @@ -45,9 +61,17 @@ func (r *remoteSQLRepositoryImpl) ListRemotes(ctx context.Context) ([]domain.Rem var remotes []domain.Remote for rows.Next() { var rem domain.Remote - if err := rows.Scan(&rem.Name, &rem.URL); err != nil { + var params sql.NullString + if err := rows.Scan(&rem.Name, &rem.URL, ¶ms); err != nil { return nil, fmt.Errorf("db: ListRemotes: scan: %w", err) } + if params.Valid { + ref, err := storage.GitRefFromParamsJSON(params.String) + if err != nil { + return nil, fmt.Errorf("db: ListRemotes: remote %s: %w", rem.Name, err) + } + rem.Ref = ref + } remotes = append(remotes, rem) } if err := rows.Err(); err != nil { diff --git a/internal/storage/domain/db/remote_test.go b/internal/storage/domain/db/remote_test.go new file mode 100644 index 000000000..6e102a9fb --- /dev/null +++ b/internal/storage/domain/db/remote_test.go @@ -0,0 +1,75 @@ +package db + +import ( + "context" + "regexp" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" +) + +// The proxied-server remote repository passes a ref as DOLT_REMOTE's --ref, +// verbatim, for both ref shapes; without a ref the call is the same +// three-argument add as before. +func TestRemoteSQLRepositoryAddRemoteWithRef(t *testing.T) { + const url = "git+https://example.com/repo.git" + for _, ref := range []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} { + t.Run(ref, func(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE(?, ?, ?, ?, ?)")). + WithArgs("add", "--ref", ref, "origin", url). + WillReturnResult(sqlmock.NewResult(0, 1)) + + repo := NewRemoteSQLRepository(db) + require.NoError(t, repo.AddRemoteWithRef(context.Background(), "origin", url, ref)) + require.NoError(t, mock.ExpectationsWereMet()) + }) + } + + t.Run("empty ref is a plain add", func(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE(?, ?, ?)")). + WithArgs("add", "origin", url). + WillReturnResult(sqlmock.NewResult(0, 1)) + + repo := NewRemoteSQLRepository(db) + require.NoError(t, repo.AddRemoteWithRef(context.Background(), "origin", url, " ")) + require.NoError(t, mock.ExpectationsWereMet()) + }) +} + +// ListRemotes reads the ref back from the params column; remotes without +// the parameter list the default. +func TestRemoteSQLRepositoryListRemotesReadsRef(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + rows := sqlmock.NewRows([]string{"name", "url", "params"}). + AddRow("plain", "file:///tmp/p", nil). + AddRow("empty", "dolthub://org/repo", "{}"). + AddRow("branch", "git+https://example.com/repo.git", `{"git_ref":"refs/heads/issue-data"}`). + AddRow("unit", "git+file:///srv/ledgers", `{"git_ref":"refs/dolt/units/team-12542"}`) + mock.ExpectQuery(regexp.QuoteMeta("SELECT name, url, params FROM dolt_remotes")).WillReturnRows(rows) + + remotes, err := NewRemoteSQLRepository(db).ListRemotes(context.Background()) + require.NoError(t, err) + want := map[string]string{ + "plain": "", + "empty": "", + "branch": "refs/heads/issue-data", + "unit": "refs/dolt/units/team-12542", + } + require.Len(t, remotes, len(want)) + for _, r := range remotes { + require.Equal(t, want[r.Name], r.Ref, "remote %s", r.Name) + } + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/internal/storage/domain/remote.go b/internal/storage/domain/remote.go index c0668e860..8a78b9be1 100644 --- a/internal/storage/domain/remote.go +++ b/internal/storage/domain/remote.go @@ -3,10 +3,18 @@ package domain import ( "context" "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage/doltutil" ) type DoltRemoteUseCase interface { CreateRemote(ctx context.Context, name, url string) error + // CreateRemoteWithRef is CreateRemote for a git-backed remote whose Dolt + // data lives on the git ref ref; an empty ref is CreateRemote. The ref is + // checked against what the dolt argv boundary will accept, so a remote + // cannot be recorded on a ref it could never be routed to. + CreateRemoteWithRef(ctx context.Context, name, url, ref string) error UpdateRemote(ctx context.Context, name, url string) error DeleteRemote(ctx context.Context, name string) error ListRemotes(ctx context.Context) ([]Remote, error) @@ -15,10 +23,14 @@ type DoltRemoteUseCase interface { type Remote struct { Name string URL string + // Ref is the git ref a git-backed remote keeps its Dolt data on when one + // was recorded; empty means Dolt's default, refs/dolt/data. + Ref string } type RemoteSQLRepository interface { AddRemote(ctx context.Context, name, url string) error + AddRemoteWithRef(ctx context.Context, name, url, ref string) error RemoveRemote(ctx context.Context, name string) error ListRemotes(ctx context.Context) ([]Remote, error) } @@ -46,6 +58,29 @@ func (u *doltRemoteUseCaseImpl) CreateRemote(ctx context.Context, name, url stri return nil } +func (u *doltRemoteUseCaseImpl) CreateRemoteWithRef(ctx context.Context, name, url, ref string) error { + if name == "" { + return fmt.Errorf("CreateRemoteWithRef: name must not be empty") + } + if url == "" { + return fmt.Errorf("CreateRemoteWithRef: url must not be empty") + } + // The SQL write paths below bind the ref as a parameter and would happily + // record one the dolt argv boundary later refuses, which would leave a + // remote that can be created but never pushed, diagnosed only at push time. + // This is the write seam for new remotes, so the recordable ref set is + // narrowed to the routable one here, beside the other argument checks. + // UpdateRemote deliberately does not re-check: it carries and restores refs + // that are already recorded, and must not become unable to put one back. + if err := doltutil.ValidateGitDataRefArg(strings.TrimSpace(ref)); err != nil { + return fmt.Errorf("CreateRemoteWithRef %s: invalid git data ref: %w", name, err) + } + if err := u.remoteRepo.AddRemoteWithRef(ctx, name, url, ref); err != nil { + return fmt.Errorf("CreateRemoteWithRef %s: %w", name, err) + } + return nil +} + func (u *doltRemoteUseCaseImpl) UpdateRemote(ctx context.Context, name, url string) error { if name == "" { return fmt.Errorf("UpdateRemote: name must not be empty") @@ -54,23 +89,34 @@ func (u *doltRemoteUseCaseImpl) UpdateRemote(ctx context.Context, name, url stri return fmt.Errorf("UpdateRemote: url must not be empty") } // Dolt has no atomic remote update, so this is remove-then-add. Capture - // the old URL first so a failed add can restore the remote instead of - // leaving it deleted (bd-6dnrw.44 P3). - var oldURL string - if remotes, err := u.remoteRepo.ListRemotes(ctx); err == nil { - for _, rem := range remotes { - if rem.Name == name { - oldURL = rem.URL - break - } + // the old URL and ref first so a failed add can restore the remote + // instead of leaving it deleted (bd-6dnrw.44 P3), and so a URL change + // keeps the remote's git data ref. A listing that fails stops the update + // before anything is removed: without it the ref could not be kept. + remotes, err := u.remoteRepo.ListRemotes(ctx) + if err != nil { + return fmt.Errorf("UpdateRemote %s: list remotes before replacing: %w", name, err) + } + var oldURL, oldRef string + for _, rem := range remotes { + if rem.Name == name { + oldURL = rem.URL + oldRef = rem.Ref + break } } + // A git data ref belongs to git-backed remotes only; Dolt refuses it on + // any other scheme, so a move to one drops it. + newRef := oldRef + if !doltutil.IsGitProtocolURL(url) { + newRef = "" + } if err := u.remoteRepo.RemoveRemote(ctx, name); err != nil { return fmt.Errorf("UpdateRemote %s: remove: %w", name, err) } - if err := u.remoteRepo.AddRemote(ctx, name, url); err != nil { + if err := u.remoteRepo.AddRemoteWithRef(ctx, name, url, newRef); err != nil { if oldURL != "" { - if restoreErr := u.remoteRepo.AddRemote(ctx, name, oldURL); restoreErr != nil { + if restoreErr := u.remoteRepo.AddRemoteWithRef(ctx, name, oldURL, oldRef); restoreErr != nil { return fmt.Errorf("UpdateRemote %s: add: %w (restoring previous URL %s also failed: %v)", name, err, oldURL, restoreErr) } return fmt.Errorf("UpdateRemote %s: add: %w (previous URL %s restored)", name, err, oldURL) diff --git a/internal/storage/domain/remote_test.go b/internal/storage/domain/remote_test.go new file mode 100644 index 000000000..8d7db75d2 --- /dev/null +++ b/internal/storage/domain/remote_test.go @@ -0,0 +1,143 @@ +package domain + +import ( + "context" + "errors" + "reflect" + "testing" +) + +type fakeRemoteRepo struct { + remotes []Remote + calls []string + failAdd string + listErr error +} + +func (f *fakeRemoteRepo) AddRemote(ctx context.Context, name, url string) error { + return f.AddRemoteWithRef(ctx, name, url, "") +} + +func (f *fakeRemoteRepo) AddRemoteWithRef(_ context.Context, name, url, ref string) error { + f.calls = append(f.calls, "add "+name+" "+url+" ref="+ref) + if url == f.failAdd { + return errors.New("refused") + } + f.remotes = append(f.remotes, Remote{Name: name, URL: url, Ref: ref}) + return nil +} + +func (f *fakeRemoteRepo) RemoveRemote(_ context.Context, name string) error { + f.calls = append(f.calls, "remove "+name) + kept := f.remotes[:0] + for _, r := range f.remotes { + if r.Name != name { + kept = append(kept, r) + } + } + f.remotes = kept + return nil +} + +func (f *fakeRemoteRepo) ListRemotes(context.Context) ([]Remote, error) { + if f.listErr != nil { + return nil, f.listErr + } + return append([]Remote(nil), f.remotes...), nil +} + +// A remote's git data ref survives UpdateRemote: the replacement carries it, +// and so does the restore after a failed replacement. Both ref shapes. +func TestUpdateRemoteKeepsGitDataRef(t *testing.T) { + for _, ref := range []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} { + t.Run(ref, func(t *testing.T) { + repo := &fakeRemoteRepo{remotes: []Remote{{Name: "origin", URL: "git+https://example.com/old.git", Ref: ref}}} + uc := NewDoltRemoteUseCase(repo) + if err := uc.UpdateRemote(context.Background(), "origin", "git+https://example.com/new.git"); err != nil { + t.Fatalf("UpdateRemote: %v", err) + } + want := []string{"remove origin", "add origin git+https://example.com/new.git ref=" + ref} + if !reflect.DeepEqual(repo.calls, want) { + t.Fatalf("calls = %v, want %v", repo.calls, want) + } + }) + } + + t.Run("restore after a failed add keeps the ref", func(t *testing.T) { + repo := &fakeRemoteRepo{ + remotes: []Remote{{Name: "origin", URL: "git+https://example.com/old.git", Ref: "refs/dolt/units/team-12542"}}, + failAdd: "git+https://example.com/new.git", + } + uc := NewDoltRemoteUseCase(repo) + if err := uc.UpdateRemote(context.Background(), "origin", "git+https://example.com/new.git"); err == nil { + t.Fatal("UpdateRemote should report the failed add") + } + if len(repo.remotes) != 1 || repo.remotes[0].Ref != "refs/dolt/units/team-12542" || repo.remotes[0].URL != "git+https://example.com/old.git" { + t.Fatalf("restored remote = %+v, want the old URL on the old ref", repo.remotes) + } + }) + + t.Run("a failed listing stops the update before removal", func(t *testing.T) { + repo := &fakeRemoteRepo{ + remotes: []Remote{{Name: "origin", URL: "git+https://example.com/old.git", Ref: "refs/dolt/units/team-12542"}}, + listErr: errors.New("server unreachable"), + } + uc := NewDoltRemoteUseCase(repo) + if err := uc.UpdateRemote(context.Background(), "origin", "git+https://example.com/new.git"); err == nil { + t.Fatal("UpdateRemote should fail when the listing fails") + } + if len(repo.calls) != 0 { + t.Fatalf("nothing may be removed or added after a failed listing, got %v", repo.calls) + } + }) + + t.Run("a move to a non-git URL drops the ref", func(t *testing.T) { + repo := &fakeRemoteRepo{remotes: []Remote{{Name: "origin", URL: "git+https://example.com/old.git", Ref: "refs/dolt/units/team-12542"}}} + uc := NewDoltRemoteUseCase(repo) + if err := uc.UpdateRemote(context.Background(), "origin", "dolthub://org/repo"); err != nil { + t.Fatalf("UpdateRemote: %v", err) + } + if want := []string{"remove origin", "add origin dolthub://org/repo ref="}; !reflect.DeepEqual(repo.calls, want) { + t.Fatalf("calls = %v, want %v", repo.calls, want) + } + }) + + t.Run("remote on the default ref stays on it", func(t *testing.T) { + repo := &fakeRemoteRepo{remotes: []Remote{{Name: "origin", URL: "file:///tmp/old"}}} + uc := NewDoltRemoteUseCase(repo) + if err := uc.UpdateRemote(context.Background(), "origin", "file:///tmp/new"); err != nil { + t.Fatalf("UpdateRemote: %v", err) + } + if want := []string{"remove origin", "add origin file:///tmp/new ref="}; !reflect.DeepEqual(repo.calls, want) { + t.Fatalf("calls = %v, want %v", repo.calls, want) + } + }) +} + +func TestCreateRemoteWithRefValidatesArguments(t *testing.T) { + repo := &fakeRemoteRepo{} + uc := NewDoltRemoteUseCase(repo) + if err := uc.CreateRemoteWithRef(context.Background(), "", "file:///tmp/x", "refs/heads/issue-data"); err == nil { + t.Error("empty name should be refused") + } + if err := uc.CreateRemoteWithRef(context.Background(), "origin", "", "refs/heads/issue-data"); err == nil { + t.Error("empty url should be refused") + } + // A ref the dolt argv boundary will refuse must not be recordable here: it + // would create a remote that can never be pushed and only says so at push + // time. Refused before the repository is touched, so nothing is recorded. + for _, ref := range []string{"-x", "refs/heads/issue data", "refs/heads/issue\tdata"} { + if err := uc.CreateRemoteWithRef(context.Background(), "origin", "git+file:///srv/ledgers", ref); err == nil { + t.Errorf("CreateRemoteWithRef with ref %q should be refused", ref) + } + } + if len(repo.calls) != 0 { + t.Fatalf("a refused CreateRemoteWithRef reached the repository: calls = %v", repo.calls) + } + if err := uc.CreateRemoteWithRef(context.Background(), "origin", "git+file:///srv/ledgers", "refs/dolt/units/k"); err != nil { + t.Fatalf("CreateRemoteWithRef: %v", err) + } + if want := []string{"add origin git+file:///srv/ledgers ref=refs/dolt/units/k"}; !reflect.DeepEqual(repo.calls, want) { + t.Fatalf("calls = %v, want %v", repo.calls, want) + } +} diff --git a/internal/storage/embeddeddolt/git_ref_pin_test.go b/internal/storage/embeddeddolt/git_ref_pin_test.go new file mode 100644 index 000000000..8b151fe61 --- /dev/null +++ b/internal/storage/embeddeddolt/git_ref_pin_test.go @@ -0,0 +1,26 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "testing" + + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" + "github.com/dolthub/dolt/go/store/blobstore" + + "github.com/steveyegge/beads/internal/storage" +) + +// The storage package pins two Dolt literals so that the binary does not +// import Dolt outside the storage backends. A Dolt bump that renamed either +// would make every ref remote read as the default and route to +// refs/dolt/data silently; this keeps the pins honest against the module +// the engine is built from. +func TestGitRefPinsMatchDolt(t *testing.T) { + if storage.GitRefParam != dbfactory.GitRefParam { + t.Errorf("storage.GitRefParam = %q, dolt dbfactory.GitRefParam = %q", storage.GitRefParam, dbfactory.GitRefParam) + } + if storage.DefaultGitDataRef != blobstore.DoltDataRef { + t.Errorf("storage.DefaultGitDataRef = %q, dolt blobstore.DoltDataRef = %q", storage.DefaultGitDataRef, blobstore.DoltDataRef) + } +} diff --git a/internal/storage/embeddeddolt/remote_git_ref_test.go b/internal/storage/embeddeddolt/remote_git_ref_test.go new file mode 100644 index 000000000..828c9c181 --- /dev/null +++ b/internal/storage/embeddeddolt/remote_git_ref_test.go @@ -0,0 +1,185 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// runGit runs git in dir and fails the test on error. +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + // Repository-local hooks path, so a developer's global core.hooksPath + // cannot run inside the fixture repositories (engdocs/TESTING.md). + cmd.Env = append(os.Environ(), + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=bd-test", "GIT_AUTHOR_EMAIL=bd-test@example.com", + "GIT_COMMITTER_NAME=bd-test", "GIT_COMMITTER_EMAIL=bd-test@example.com", + "GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=core.hooksPath", "GIT_CONFIG_VALUE_0="+filepath.Join(dir, ".git", "hooks"), + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return string(out) +} + +// newBareGitRemote creates a bare repository with one commit on main, the +// shape a git host presents before any Dolt data has been pushed to it. +func newBareGitRemote(t *testing.T, name string) string { + t.Helper() + base := t.TempDir() + bare := filepath.Join(base, name) + runGit(t, base, "init", "--bare", "-b", "main", bare) + seed := filepath.Join(base, "seed") + if err := os.MkdirAll(seed, 0o755); err != nil { + t.Fatal(err) + } + runGit(t, seed, "init", "-b", "main") + runGit(t, seed, "commit", "--allow-empty", "-m", "init") + runGit(t, seed, "push", bare, "main") + return bare +} + +// lsRemoteRef returns the object a ref points at on the bare repository, or +// "" when the ref does not exist there. +func lsRemoteRef(t *testing.T, bare, ref string) string { + t.Helper() + return strings.TrimSpace(runGit(t, bare, "ls-remote", bare, ref)) +} + +// The engine, not a mock, is the authority on the SQL shapes above: a remote +// added with a ref lists that ref back from dolt_remotes and from +// repo_state.json, for a branch and for a ref outside refs/heads/; a remote +// added without one lists the default; Dolt refuses a ref on a non-git URL +// and the refusal reaches the caller; a push lands on the custom ref and +// never on refs/dolt/data; a clone with the ref records it on origin. +func TestRemoteGitDataRefOnEngine(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + ctx := context.Background() + te := newTestEnv(t, "gref") + + shapes := map[string]string{ + "branch": "refs/heads/issue-data", + "unit": "refs/dolt/units/team-12542", + } + bares := map[string]string{} + for name, ref := range shapes { + bares[name] = newBareGitRemote(t, name) + if err := te.store.AddRemoteWithRef(ctx, name, "git+file://"+bares[name], ref); err != nil { + t.Fatalf("AddRemoteWithRef(%s, %s): %v", name, ref, err) + } + } + plainDir := t.TempDir() + if err := te.store.AddRemote(ctx, "plain", "file://"+plainDir); err != nil { + t.Fatalf("AddRemote(plain): %v", err) + } + + t.Run("listed from SQL and from disk", func(t *testing.T) { + remotes, err := te.store.ListRemotes(ctx) + if err != nil { + t.Fatalf("ListRemotes: %v", err) + } + assertRefs(t, "ListRemotes", remotes, shapes) + + persisted, err := doltutil.PersistedRemotes(filepath.Join(te.dataDir, te.database)) + if err != nil { + t.Fatalf("PersistedRemotes: %v", err) + } + assertRefs(t, "PersistedRemotes", persisted, shapes) + }) + + t.Run("ref on a non-git remote is Dolt's error, not a silent default", func(t *testing.T) { + err := te.store.AddRemoteWithRef(ctx, "bad", "file://"+t.TempDir(), "refs/heads/issue-data") + if err == nil { + t.Fatal("AddRemoteWithRef on file:// with a ref = nil, want error") + } + if !strings.Contains(err.Error(), "git remotes") { + t.Errorf("error should be Dolt's --ref refusal, got: %v", err) + } + if ok, _ := te.store.HasRemote(ctx, "bad"); ok { + t.Error("refused remote must not exist") + } + }) + + for name, ref := range shapes { + t.Run("push and clone on "+ref, func(t *testing.T) { + bare := bares[name] + if err := te.store.PushRemote(ctx, name, false); err != nil { + t.Fatalf("PushRemote(%s): %v", name, err) + } + if got := lsRemoteRef(t, bare, ref); got == "" { + t.Fatalf("push did not create %s on the git remote", ref) + } + if got := lsRemoteRef(t, bare, storage.DefaultGitDataRef); got != "" { + t.Errorf("push created %s (%s) although the remote was added with --ref %s", storage.DefaultGitDataRef, got, ref) + } + if got := lsRemoteRef(t, bare, "refs/heads/main"); got == "" { + t.Error("refs/heads/main disappeared from the git remote") + } + + cloneDir := t.TempDir() + boot, bootCleanup, err := embeddeddolt.OpenSQL(ctx, cloneDir, "", "") + if err != nil { + t.Fatalf("OpenSQL for clone: %v", err) + } + cloneErr := versioncontrolops.DoltCloneWithRef(ctx, boot, "git+file://"+bare, "cloned", "", ref) + _ = bootCleanup() + if cloneErr != nil { + t.Fatalf("DoltCloneWithRef: %v", cloneErr) + } + persisted, err := doltutil.PersistedRemotes(filepath.Join(cloneDir, "cloned")) + if err != nil { + t.Fatalf("PersistedRemotes on clone: %v", err) + } + assertRefs(t, "clone repo_state.json", persisted, map[string]string{"origin": ref}) + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, cloneDir, "cloned", "main") + if err != nil { + t.Fatalf("OpenSQL on clone: %v", err) + } + defer func() { _ = cleanup() }() + listed, err := versioncontrolops.ListRemotes(ctx, db) + if err != nil { + t.Fatalf("ListRemotes on clone: %v", err) + } + assertRefs(t, "clone dolt_remotes", listed, map[string]string{"origin": ref}) + }) + } +} + +// assertRefs checks that every named remote in want lists exactly that ref +// and that the remote named plain, when present, lists the default. +func assertRefs(t *testing.T, where string, remotes []storage.RemoteInfo, want map[string]string) { + t.Helper() + got := map[string]string{} + for _, r := range remotes { + got[r.Name] = r.Ref + } + for name, ref := range want { + if _, ok := got[name]; !ok { + t.Errorf("%s: remote %s missing from %+v", where, name, remotes) + continue + } + if got[name] != ref { + t.Errorf("%s: remote %s Ref = %q, want %q", where, name, got[name], ref) + } + } + if ref, ok := got["plain"]; ok && ref != "" { + t.Errorf("%s: remote plain Ref = %q, want empty", where, ref) + } +} diff --git a/internal/storage/embeddeddolt/version_control.go b/internal/storage/embeddeddolt/version_control.go index 47f55cc87..7a5b4d95a 100644 --- a/internal/storage/embeddeddolt/version_control.go +++ b/internal/storage/embeddeddolt/version_control.go @@ -198,9 +198,14 @@ func (s *EmbeddedDoltStore) CommitMergeResolution(ctx context.Context, message s } func (s *EmbeddedDoltStore) AddRemote(ctx context.Context, name, url string) error { + return s.AddRemoteWithRef(ctx, name, url, "") +} + +// AddRemoteWithRef adds a remote whose Dolt data lives on the git ref ref; +// see storage.RemoteStore. +func (s *EmbeddedDoltStore) AddRemoteWithRef(ctx context.Context, name, url, ref string) error { return s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { - _, err := db.ExecContext(ctx, "CALL DOLT_REMOTE('add', ?, ?)", name, url) - return err + return versioncontrolops.AddRemote(ctx, db, name, url, ref) }) } diff --git a/internal/storage/remote.go b/internal/storage/remote.go index c2db8386c..c92e0f28b 100644 --- a/internal/storage/remote.go +++ b/internal/storage/remote.go @@ -5,6 +5,10 @@ import "context" // RemoteStore provides remote management and push/pull/fetch operations. type RemoteStore interface { AddRemote(ctx context.Context, name, url string) error + // AddRemoteWithRef is AddRemote for a git-backed remote whose Dolt data + // lives on the git ref ref instead of DefaultGitDataRef. An empty ref is + // AddRemote. + AddRemoteWithRef(ctx context.Context, name, url, ref string) error RemoveRemote(ctx context.Context, name string) error HasRemote(ctx context.Context, name string) (bool, error) ListRemotes(ctx context.Context) ([]RemoteInfo, error) diff --git a/internal/storage/remote_ref.go b/internal/storage/remote_ref.go new file mode 100644 index 000000000..30d10365c --- /dev/null +++ b/internal/storage/remote_ref.go @@ -0,0 +1,69 @@ +package storage + +import ( + "encoding/json" + "fmt" + "strings" +) + +// A git-backed Dolt remote keeps its data on one git ref of the remote +// repository. Dolt records the ref as the remote parameter GitRefParam and +// falls back to DefaultGitDataRef when the parameter is absent. +const ( + GitRefParam = "git_ref" + DefaultGitDataRef = "refs/dolt/data" +) + +// EffectiveGitDataRef returns ref, or DefaultGitDataRef when ref is empty. +func EffectiveGitDataRef(ref string) string { + if ref = strings.TrimSpace(ref); ref != "" { + return ref + } + return DefaultGitDataRef +} + +// RemoteRefsMatch reports whether two git data refs name the same ref. An +// empty ref stands for DefaultGitDataRef, so an unset ref and an explicit +// default compare equal. +func RemoteRefsMatch(a, b string) bool { + return EffectiveGitDataRef(a) == EffectiveGitDataRef(b) +} + +// GitRefFromParamsJSON extracts GitRefParam from a dolt_remotes params value. +// "", "null", an empty object, and an object without the key mean the +// default ref; anything that is not a JSON object is an error rather than a +// silent default. +func GitRefFromParamsJSON(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "null" { + return "", nil + } + var params map[string]any + if err := json.Unmarshal([]byte(raw), ¶ms); err != nil { + return "", fmt.Errorf("parse dolt_remotes params %q: %w", raw, err) + } + return GitRefFromParams(params) +} + +// GitRefFromParams extracts GitRefParam from an already-decoded dolt_remotes +// params object. It is the one reader of that key: GitRefFromParamsJSON uses +// it for the SQL params column and doltutil.PersistedRemotes for the same +// object in repo_state.json, so the two planes cannot drift. +// +// An absent key and a JSON null mean the default ref. A key that is present +// but is not a string is an error, for the same reason a params value that is +// not a JSON object is: a recorded ref bd cannot read could be a custom ref +// the mirror is pinned to, and reading it as the default would push the +// remote's data onto DefaultGitDataRef. Dolt only ever writes a string here, +// so this is the fail-closed edge of that invariant, not a live shape. +func GitRefFromParams(params map[string]any) (string, error) { + value, ok := params[GitRefParam] + if !ok || value == nil { + return "", nil + } + ref, ok := value.(string) + if !ok { + return "", fmt.Errorf("remote parameter %s is %T, not a string", GitRefParam, value) + } + return strings.TrimSpace(ref), nil +} diff --git a/internal/storage/remote_ref_test.go b/internal/storage/remote_ref_test.go new file mode 100644 index 000000000..bcd60b719 --- /dev/null +++ b/internal/storage/remote_ref_test.go @@ -0,0 +1,114 @@ +package storage + +import ( + "strings" + "testing" +) + +// An unset ref and Dolt's default name the same ref; any other value is +// compared verbatim, so a branch and a ref outside refs/heads/ are distinct +// from each other and from the default. +func TestRemoteRefsMatch(t *testing.T) { + tests := []struct { + a, b string + want bool + }{ + {"", "", true}, + {"", DefaultGitDataRef, true}, + {DefaultGitDataRef, "", true}, + {" refs/dolt/data ", "", true}, + {"refs/heads/issue-data", "refs/heads/issue-data", true}, + {"refs/dolt/units/team-12542", "refs/dolt/units/team-12542", true}, + {"refs/heads/issue-data", "", false}, + {"refs/dolt/units/team-12542", "", false}, + {"refs/heads/issue-data", "refs/dolt/units/team-12542", false}, + {"refs/heads/issue-data", "refs/heads/issue-data-2", false}, + } + for _, tt := range tests { + if got := RemoteRefsMatch(tt.a, tt.b); got != tt.want { + t.Errorf("RemoteRefsMatch(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want) + } + } +} + +// A params object carries the ref as a string or not at all. A present value +// of any other JSON type is an error rather than the default ref: reading it as +// the default is how a remote pinned to a custom ref gets pushed onto +// refs/dolt/data, which is exactly the failure this parameter exists to +// prevent. An absent key and an explicit null are the genuine "no ref" shapes. +func TestGitRefFromParams(t *testing.T) { + for _, tt := range []struct { + name string + params map[string]any + want string + }{ + {"absent", map[string]any{"aws-region": "us-east-1"}, ""}, + {"empty object", map[string]any{}, ""}, + {"nil map", nil, ""}, + {"null", map[string]any{GitRefParam: nil}, ""}, + {"empty string", map[string]any{GitRefParam: ""}, ""}, + {"trimmed", map[string]any{GitRefParam: " refs/dolt/units/k "}, "refs/dolt/units/k"}, + {"branch", map[string]any{GitRefParam: "refs/heads/issue-data"}, "refs/heads/issue-data"}, + } { + got, err := GitRefFromParams(tt.params) + if err != nil || got != tt.want { + t.Errorf("GitRefFromParams(%s) = %q, %v; want %q, nil", tt.name, got, err, tt.want) + } + } + + for _, tt := range []struct { + name string + value any + }{ + {"array", []any{"refs/dolt/units/x"}}, + {"number", float64(7)}, + {"bool", true}, + {"object", map[string]any{"ref": "refs/dolt/units/x"}}, + } { + got, err := GitRefFromParams(map[string]any{GitRefParam: tt.value}) + if err == nil { + t.Errorf("GitRefFromParams(%s) = %q, nil; want an error, not the default ref", tt.name, got) + continue + } + if !strings.Contains(err.Error(), GitRefParam) { + t.Errorf("GitRefFromParams(%s) error %q does not name %s", tt.name, err, GitRefParam) + } + } +} + +// GitRefFromParamsJSON reads the same key out of the SQL params column, so it +// must refuse the same shapes: the two readers are one implementation. +func TestGitRefFromParamsJSONRefusesNonStringRef(t *testing.T) { + for _, raw := range []string{ + `{"git_ref":["refs/dolt/units/x"]}`, + `{"git_ref":7}`, + `{"git_ref":true}`, + `{"git_ref":{"ref":"refs/dolt/units/x"}}`, + } { + if got, err := GitRefFromParamsJSON(raw); err == nil { + t.Errorf("GitRefFromParamsJSON(%s) = %q, nil; want an error", raw, got) + } + } + for _, tt := range []struct{ raw, want string }{ + {"", ""}, + {"null", ""}, + {`{}`, ""}, + {`{"git_ref":null}`, ""}, + {`{"aws-region":"us-east-1"}`, ""}, + {`{"git_ref":"refs/heads/issue-data"}`, "refs/heads/issue-data"}, + } { + got, err := GitRefFromParamsJSON(tt.raw) + if err != nil || got != tt.want { + t.Errorf("GitRefFromParamsJSON(%s) = %q, %v; want %q, nil", tt.raw, got, err, tt.want) + } + } +} + +func TestEffectiveGitDataRef(t *testing.T) { + if got := EffectiveGitDataRef(""); got != DefaultGitDataRef { + t.Errorf("EffectiveGitDataRef(\"\") = %q, want %q", got, DefaultGitDataRef) + } + if got := EffectiveGitDataRef(" refs/dolt/units/k "); got != "refs/dolt/units/k" { + t.Errorf("EffectiveGitDataRef trims but does not rewrite: got %q", got) + } +} diff --git a/internal/storage/versioncontrolops/clone.go b/internal/storage/versioncontrolops/clone.go index 3f17f7dbc..f3477081b 100644 --- a/internal/storage/versioncontrolops/clone.go +++ b/internal/storage/versioncontrolops/clone.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "strings" ) // DoltClone clones a Dolt database from a remote URL. @@ -12,12 +13,27 @@ import ( // If user is non-empty, authenticates with that user. Dolt reads the remote // password from DOLT_REMOTE_PASSWORD. func DoltClone(ctx context.Context, conn DBConn, remoteURL, database, user string) error { - query := "CALL DOLT_CLONE(?, ?)" - args := []any{remoteURL, database} + return DoltCloneWithRef(ctx, conn, remoteURL, database, user, "") +} + +// DoltCloneWithRef is DoltClone for a git-backed remote whose Dolt data lives +// on the git ref ref. A non-empty ref is passed as DOLT_CLONE's --ref option; +// Dolt reads the data from that ref and records it on the clone's origin +// remote, so later push and pull use it. An empty ref is DoltClone. +func DoltCloneWithRef(ctx context.Context, conn DBConn, remoteURL, database, user, ref string) error { + var parts []string + var args []any if user != "" { - query = "CALL DOLT_CLONE('--user', ?, ?, ?)" - args = []any{user, remoteURL, database} + parts = append(parts, "'--user', ?") + args = append(args, user) } + if ref = strings.TrimSpace(ref); ref != "" { + parts = append(parts, "'--ref', ?") + args = append(args, ref) + } + parts = append(parts, "?, ?") + args = append(args, remoteURL, database) + query := "CALL DOLT_CLONE(" + strings.Join(parts, ", ") + ")" // GH#4272: the initial fetch runs git hooks just like push/pull; disable // them for the clone window too (see remotes.go for the full rationale). diff --git a/internal/storage/versioncontrolops/clone_test.go b/internal/storage/versioncontrolops/clone_test.go index 830663ee4..342fb3b33 100644 --- a/internal/storage/versioncontrolops/clone_test.go +++ b/internal/storage/versioncontrolops/clone_test.go @@ -27,6 +27,49 @@ func TestDoltCloneWithoutUser(t *testing.T) { } } +// A ref reaches DOLT_CLONE as --ref, verbatim, for both ref shapes. +func TestDoltCloneWithRef(t *testing.T) { + for _, ref := range gitDataRefShapes { + t.Run(ref, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_CLONE('--ref', ?, ?, ?)")). + WithArgs(ref, "git+https://example.com/repo.git", "beads"). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := DoltCloneWithRef(context.Background(), db, "git+https://example.com/repo.git", "beads", "", ref); err != nil { + t.Fatalf("DoltCloneWithRef: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestDoltCloneWithUserAndRef(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_CLONE('--user', ?, '--ref', ?, ?, ?)")). + WithArgs("alice", "refs/dolt/units/team-12542", "git+https://example.com/repo.git", "beads"). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := DoltCloneWithRef(context.Background(), db, "git+https://example.com/repo.git", "beads", "alice", "refs/dolt/units/team-12542"); err != nil { + t.Fatalf("DoltCloneWithRef: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestDoltCloneWithUser(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { diff --git a/internal/storage/versioncontrolops/remotes.go b/internal/storage/versioncontrolops/remotes.go index fdf373004..9ef349a1e 100644 --- a/internal/storage/versioncontrolops/remotes.go +++ b/internal/storage/versioncontrolops/remotes.go @@ -2,14 +2,19 @@ package versioncontrolops import ( "context" + "database/sql" "fmt" + "strings" "github.com/steveyegge/beads/internal/storage" ) -// ListRemotes returns all configured Dolt remotes (name and URL). +// ListRemotes returns all configured Dolt remotes: name, URL, and the git ref +// a git-backed remote keeps its data on when one was set. The ref is the +// git_ref key of the dolt_remotes params column, a JSON object of the +// remote's parameters; a remote without the key lists an empty Ref. func ListRemotes(ctx context.Context, db DBConn) ([]storage.RemoteInfo, error) { - rows, err := db.QueryContext(ctx, "SELECT name, url FROM dolt_remotes") + rows, err := db.QueryContext(ctx, "SELECT name, url, params FROM dolt_remotes") if err != nil { return nil, fmt.Errorf("list remotes: %w", err) } @@ -18,14 +23,35 @@ func ListRemotes(ctx context.Context, db DBConn) ([]storage.RemoteInfo, error) { var remotes []storage.RemoteInfo for rows.Next() { var r storage.RemoteInfo - if err := rows.Scan(&r.Name, &r.URL); err != nil { + var params sql.NullString + if err := rows.Scan(&r.Name, &r.URL, ¶ms); err != nil { return nil, fmt.Errorf("scan remote: %w", err) } + if params.Valid { + ref, err := storage.GitRefFromParamsJSON(params.String) + if err != nil { + return nil, fmt.Errorf("remote %s: %w", r.Name, err) + } + r.Ref = ref + } remotes = append(remotes, r) } return remotes, rows.Err() } +// AddRemote adds a Dolt remote. A non-empty ref is passed as DOLT_REMOTE's +// --ref option, which Dolt records as the remote's git_ref parameter and +// accepts for git-backed remotes only; for any other scheme Dolt's own +// refusal is returned unchanged. +func AddRemote(ctx context.Context, db DBConn, name, url, ref string) error { + if ref = strings.TrimSpace(ref); ref == "" { + _, err := db.ExecContext(ctx, "CALL DOLT_REMOTE('add', ?, ?)", name, url) + return err + } + _, err := db.ExecContext(ctx, "CALL DOLT_REMOTE('add', '--ref', ?, ?, ?)", ref, name, url) + return err +} + // RemoveRemote removes a configured Dolt remote. func RemoveRemote(ctx context.Context, db DBConn, name string) error { if _, err := db.ExecContext(ctx, "CALL DOLT_REMOTE('remove', ?)", name); err != nil { diff --git a/internal/storage/versioncontrolops/remotes_test.go b/internal/storage/versioncontrolops/remotes_test.go new file mode 100644 index 000000000..15aaf81de --- /dev/null +++ b/internal/storage/versioncontrolops/remotes_test.go @@ -0,0 +1,132 @@ +package versioncontrolops + +import ( + "context" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +// The two ref shapes every ref test in this change covers: a branch, for a +// git host that only accepts pushes under refs/heads/, and a ref elsewhere +// in the namespace, for several databases kept in one repository. +var gitDataRefShapes = []string{"refs/heads/issue-data", "refs/dolt/units/team-12542"} + +// Without a ref the add is the same two-argument DOLT_REMOTE call every +// existing remote was created with. +func TestAddRemoteWithoutRef(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE('add', ?, ?)")). + WithArgs("origin", "git+https://example.com/repo.git"). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := AddRemote(context.Background(), db, "origin", "git+https://example.com/repo.git", " "); err != nil { + t.Fatalf("AddRemote: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// With a ref the add carries --ref, and the value reaches Dolt verbatim for +// both shapes. +func TestAddRemoteWithRef(t *testing.T) { + for _, ref := range gitDataRefShapes { + t.Run(ref, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_REMOTE('add', '--ref', ?, ?, ?)")). + WithArgs(ref, "origin", "git+https://example.com/repo.git"). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := AddRemote(context.Background(), db, "origin", "git+https://example.com/repo.git", ref); err != nil { + t.Fatalf("AddRemote: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) + } +} + +// ListRemotes reads the ref back out of the params column. A NULL, the text +// "null" a sql-server sends for a remote without parameters, an empty +// object, and an object without git_ref all list as the default (empty Ref); +// a recorded git_ref comes back verbatim. +func TestListRemotesReadsGitRefParam(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + rows := sqlmock.NewRows([]string{"name", "url", "params"}). + AddRow("null-params", "file:///tmp/a", nil). + AddRow("null-text", "file:///tmp/b", "null"). + AddRow("empty-params", "dolthub://org/repo", "{}"). + AddRow("other-params", "aws://bucket/repo", `{"aws-region":"us-east-1"}`). + AddRow("branch", "git+https://example.com/repo.git", `{"git_ref":"refs/heads/issue-data"}`). + AddRow("unit", "git+file:///srv/ledgers", `{"git_ref":"refs/dolt/units/team-12542"}`) + mock.ExpectQuery(regexp.QuoteMeta("SELECT name, url, params FROM dolt_remotes")).WillReturnRows(rows) + + remotes, err := ListRemotes(context.Background(), db) + if err != nil { + t.Fatalf("ListRemotes: %v", err) + } + want := map[string]string{ + "null-params": "", + "null-text": "", + "empty-params": "", + "other-params": "", + "branch": "refs/heads/issue-data", + "unit": "refs/dolt/units/team-12542", + } + if len(remotes) != len(want) { + t.Fatalf("ListRemotes returned %d remotes, want %d: %+v", len(remotes), len(want), remotes) + } + for _, r := range remotes { + if r.Ref != want[r.Name] { + t.Errorf("remote %s: Ref = %q, want %q", r.Name, r.Ref, want[r.Name]) + } + } + if remotes[0].URL != "file:///tmp/a" { + t.Errorf("URL still scanned: got %q", remotes[0].URL) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// A params value that is not a JSON object is an error naming the remote, +// never a silent fall-back to the default ref: routing a custom-ref remote +// through the default would push its data to the wrong ref. +func TestListRemotesRejectsMalformedParams(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + rows := sqlmock.NewRows([]string{"name", "url", "params"}). + AddRow("origin", "git+https://example.com/repo.git", "not-json") + mock.ExpectQuery(regexp.QuoteMeta("SELECT name, url, params FROM dolt_remotes")).WillReturnRows(rows) + + _, err = ListRemotes(context.Background(), db) + if err == nil { + t.Fatal("ListRemotes with malformed params = nil error, want error") + } + if !strings.Contains(err.Error(), "origin") || !strings.Contains(err.Error(), "params") { + t.Errorf("error should name the remote and the params column: %v", err) + } +} diff --git a/internal/storage/versioned.go b/internal/storage/versioned.go index 5722df688..46a745cf5 100644 --- a/internal/storage/versioned.go +++ b/internal/storage/versioned.go @@ -39,6 +39,10 @@ type Conflict struct { type RemoteInfo struct { Name string `json:"name"` // Remote name (e.g., "town-beta") URL string `json:"url"` // Remote URL (e.g., "dolthub://org/repo") + // Ref is the git ref a git-backed remote keeps its Dolt data on, when the + // remote was added with one. Empty means DefaultGitDataRef; always empty + // for remotes that are not git-backed. + Ref string `json:"ref,omitempty"` } // SyncStatus describes the synchronization state with a peer.