From 73d72ec0fa31cf587d9f8bcbd2ac29b8046337f9 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Mon, 20 Apr 2026 18:02:33 -0700 Subject: [PATCH] feat: add --remote flag to bd dolt push/pull commands (#3211) Add PushRemote/PullRemote methods to RemoteStore interface, allowing push/pull to target a specific named remote instead of the default. Implementation: - Refactor Push/ForcePush/Pull to delegate to internal pushToRemote/ pullFromRemote methods with parameterized remote name - Parameterize credential routing (isGitProtocolRemote, shouldUseCLIForCredentials, shouldUseCLIForCloudAuth) to accept remote name instead of using s.remote - Make shouldUseCLIForCloudAuth per-scheme-aware: Azure env vars only trigger CLI routing for az:// remotes, not dolthub:// remotes - Add credentialsForRemote() helper that only returns creds when the target remote matches the default remote (prevents sending wrong creds to wrong host) - Add --remote flag to doltPushCmd and doltPullCmd in CLI - Implement PushRemote/PullRemote on EmbeddedDoltStore Backwards compatibility: - Push(ctx)/Pull(ctx)/ForcePush(ctx) signatures unchanged - No --remote flag = exact same behavior as today - PushTo/PullFrom for federation peers unchanged Closes: bd-4v3 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cmd/bd/dolt.go | 46 ++++- internal/storage/dolt/credentials.go | 70 +++++-- internal/storage/dolt/credentials_test.go | 181 +++++++++++++++--- internal/storage/dolt/git_remote_test.go | 6 +- internal/storage/dolt/store.go | 176 +++++++++-------- .../storage/embeddeddolt/version_control.go | 15 ++ internal/storage/remote.go | 2 + 7 files changed, 363 insertions(+), 133 deletions(-) diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index c18d90760..abbdf133c 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -215,7 +215,10 @@ For Hosted Dolt, set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD environment variables for authentication. Use --force to overwrite remote changes (e.g., when the remote has -uncommitted changes in its working set).`, +uncommitted changes in its working set). + +Use --remote to push to a specific named remote instead of the default. +The remote must already exist (see 'bd dolt remote add').`, Run: func(cmd *cobra.Command, args []string) { ctx := context.Background() st := getStore() @@ -224,6 +227,23 @@ uncommitted changes in its working set).`, os.Exit(1) } force, _ := cmd.Flags().GetBool("force") + remote, _ := cmd.Flags().GetString("remote") + if remote != "" { + fmt.Printf("Pushing to Dolt remote %q...\n", remote) + if err := st.PushRemote(ctx, remote, force); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + if isRemoteNotFoundErr(err) { + fmt.Fprintf(os.Stderr, "\nRemote %q is not configured.\n", remote) + fmt.Fprintln(os.Stderr, "Use 'bd dolt remote add ' to add it.") + fmt.Fprintln(os.Stderr, "Use 'bd dolt remote list' to see configured remotes.") + } else if isDivergedHistoryErr(err) { + printDivergedHistoryGuidance("push --force") + } + os.Exit(1) + } + fmt.Println("Push complete.") + return + } fmt.Println("Pushing to Dolt remote...") var pushErr error @@ -258,7 +278,10 @@ var doltPullCmd = &cobra.Command{ Requires a Dolt remote to be configured in the database directory. For Hosted Dolt, set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD environment -variables for authentication.`, +variables for authentication. + +Use --remote to pull from a specific named remote instead of the default. +The remote must already exist (see 'bd dolt remote add').`, Run: func(cmd *cobra.Command, args []string) { ctx := context.Background() st := getStore() @@ -266,6 +289,23 @@ variables for authentication.`, fmt.Fprintf(os.Stderr, "Error: no store available\n") os.Exit(1) } + remote, _ := cmd.Flags().GetString("remote") + if remote != "" { + fmt.Printf("Pulling from Dolt remote %q...\n", remote) + if err := st.PullRemote(ctx, remote); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + if isRemoteNotFoundErr(err) { + fmt.Fprintf(os.Stderr, "\nRemote %q is not configured.\n", remote) + fmt.Fprintln(os.Stderr, "Use 'bd dolt remote add ' to add it.") + fmt.Fprintln(os.Stderr, "Use 'bd dolt remote list' to see configured remotes.") + } else if isDivergedHistoryErr(err) { + printDivergedHistoryGuidance("pull") + } + os.Exit(1) + } + fmt.Println("Pull complete.") + return + } fmt.Println("Pulling from Dolt remote...") if err := st.Pull(ctx); err != nil { if isRemoteNotFoundErr(err) { @@ -1104,6 +1144,8 @@ func init() { doltSetCmd.Flags().Bool("update-config", false, "Also write to config.yaml for team-wide defaults") doltStopCmd.Flags().Bool("force", false, "Force stop the server") doltPushCmd.Flags().Bool("force", false, "Force push (overwrite remote changes)") + doltPushCmd.Flags().String("remote", "", "Push to a specific named remote instead of the default") + doltPullCmd.Flags().String("remote", "", "Pull from a specific named remote instead of the default") doltCommitCmd.Flags().StringP("message", "m", "", "Commit message (default: auto-generated)") doltCleanDatabasesCmd.Flags().Bool("dry-run", false, "Show what would be dropped without dropping") doltRemoteRemoveCmd.Flags().Bool("force", false, "Force remove even when SQL and CLI URLs conflict") diff --git a/internal/storage/dolt/credentials.go b/internal/storage/dolt/credentials.go index 5053d4450..dbb44fd14 100644 --- a/internal/storage/dolt/credentials.go +++ b/internal/storage/dolt/credentials.go @@ -521,8 +521,8 @@ func (s *DoltStore) shouldUseCLIForPeerCredentials(_ context.Context, peer strin // 2. Server is in server mode (not embedded) // 3. Local CLI directory is available // 4. The remote is configured in the local CLI directory -func (s *DoltStore) shouldUseCLIForCredentials(_ context.Context) bool { - if s.remoteUser == "" && s.remotePassword == "" { +func (s *DoltStore) shouldUseCLIForCredentials(_ context.Context, remote string, creds *remoteCredentials) bool { + if creds.empty() { return false // no credentials to pass } if !s.serverMode { @@ -535,27 +535,50 @@ func (s *DoltStore) shouldUseCLIForCredentials(_ context.Context) bool { // Only route to CLI if the remote is configured locally. // Shared server / external server modes may have CLIDir pointing // to wrong directory — FindCLIRemote returns "" in those cases. - return doltutil.FindCLIRemote(cliDir, s.remote) != "" + return doltutil.FindCLIRemote(cliDir, remote) != "" } -// cloudAuthEnvPrefixes lists environment variable prefixes used by cloud -// storage providers for authentication. When any of these are set and the -// store is in server mode, push/pull must route through a CLI subprocess -// so the dolt process inherits the current env vars. The SQL path -// (CALL DOLT_PUSH/PULL) executes inside the dolt-sql-server, which only -// has env vars from when it was started — not from the current shell. -var cloudAuthEnvPrefixes = []string{ - "AZURE_STORAGE_", // Azure Blob Storage (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, AZURE_STORAGE_SAS_TOKEN) - "AWS_", // AWS S3 (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, AWS_REGION) - "GOOGLE_", // GCS (GOOGLE_APPLICATION_CREDENTIALS) - "GCS_", // GCS alternate (GCS_CREDENTIALS_FILE) - "OCI_", // Oracle Cloud Infrastructure - "DOLT_REMOTE_", // Dolt-specific remote credentials +// cloudAuthSchemeMap maps remote URL scheme prefixes to the environment +// variable prefixes that provide credentials for that scheme. Only env vars +// relevant to the remote's scheme are checked, preventing misrouting when +// multiple remotes use different cloud providers (e.g., DoltHub + Azure). +// +// The CLI subprocess inherits the current process env, so these env vars +// reach the dolt binary. The SQL server process may not have them if it was +// started in a different context (GH#6). +var cloudAuthSchemeMap = map[string][]string{ + "az://": {"AZURE_STORAGE_"}, // Azure Blob Storage + "s3://": {"AWS_"}, // AWS S3 + "gs://": {"GOOGLE_", "GCS_"}, // Google Cloud Storage + "oci://": {"OCI_"}, // Oracle Cloud Infrastructure + "dolthub://": {"DOLT_REMOTE_"}, // DoltHub + "https://": {"DOLT_REMOTE_"}, // Hosted Dolt / DoltHub HTTPS + "http://": {"DOLT_REMOTE_"}, // Hosted Dolt HTTP +} + +// envPrefixesForRemoteURL returns the env var prefixes relevant to the +// given remote URL based on its scheme. Returns nil for unrecognized schemes +// (git-protocol remotes are handled by isGitProtocolRemote, not here). +func envPrefixesForRemoteURL(url string) []string { + for scheme, prefixes := range cloudAuthSchemeMap { + if strings.HasPrefix(url, scheme) { + return prefixes + } + } + return nil } // shouldUseCLIForCloudAuth returns true when CLI subprocess routing should -// be used for push/pull because cloud storage credentials are present in the -// environment and the store is using an external dolt-sql-server. +// be used for push/pull because cloud storage credentials relevant to this +// specific remote are present in the environment and the store is using an +// external dolt-sql-server. +// +// Unlike a global heuristic, this checks only the env var prefixes that +// match the remote's URL scheme. An Azure env var (AZURE_STORAGE_ACCOUNT) +// will trigger CLI routing for an az:// remote but NOT for a dolthub:// remote. +// +// The CLI remote URL is used for scheme detection because that is the URL +// the CLI subprocess will actually use (SQL remotes may differ due to drift). // // When bd connects to an external dolt-sql-server (server mode), CALL // DOLT_PUSH/PULL executes inside the server process. That process only has @@ -563,7 +586,7 @@ var cloudAuthEnvPrefixes = []string{ // changed) after the server started, the SQL path silently fails to // authenticate. Routing through a CLI subprocess (dolt push/pull) ensures // the child process inherits the current environment (GH#6). -func (s *DoltStore) shouldUseCLIForCloudAuth() bool { +func (s *DoltStore) shouldUseCLIForCloudAuth(remote string) bool { if !s.serverMode { return false // embedded mode: env vars are in-process } @@ -571,11 +594,16 @@ func (s *DoltStore) shouldUseCLIForCloudAuth() bool { if cliDir == "" { return false } - if doltutil.FindCLIRemote(cliDir, s.remote) == "" { + cliURL := doltutil.FindCLIRemote(cliDir, remote) + if cliURL == "" { return false } + prefixes := envPrefixesForRemoteURL(cliURL) + if len(prefixes) == 0 { + return false // unknown scheme — not a cloud remote + } for _, e := range os.Environ() { - for _, prefix := range cloudAuthEnvPrefixes { + for _, prefix := range prefixes { if strings.HasPrefix(e, prefix) { return true } diff --git a/internal/storage/dolt/credentials_test.go b/internal/storage/dolt/credentials_test.go index 4bbeb49a7..2e3bc42ed 100644 --- a/internal/storage/dolt/credentials_test.go +++ b/internal/storage/dolt/credentials_test.go @@ -301,6 +301,10 @@ func TestCredentialKeyCreatesBeadsDir(t *testing.T) { // setupCredentialTestStore creates a DoltStore with a dolt-initialized CLI directory // and "origin" remote for credential routing tests. Requires dolt CLI. func setupCredentialTestStore(t *testing.T, remoteUser, remotePassword string, serverMode, setupRemote bool) *DoltStore { + return setupCredentialTestStoreWithURL(t, remoteUser, remotePassword, serverMode, setupRemote, "origin", "https://example.com/repo") +} + +func setupCredentialTestStoreWithURL(t *testing.T, remoteUser, remotePassword string, serverMode, setupRemote bool, remoteName, remoteURL string) *DoltStore { t.Helper() tmpDir := t.TempDir() dbName := "testdb" @@ -315,7 +319,7 @@ func setupCredentialTestStore(t *testing.T, remoteUser, remotePassword string, s if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("dolt init failed: %s: %v", out, err) } - cmd = exec.Command("dolt", "remote", "add", "origin", "https://example.com/repo") + cmd = exec.Command("dolt", "remote", "add", remoteName, remoteURL) cmd.Dir = dbDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("dolt remote add failed: %s: %v", out, err) @@ -328,7 +332,7 @@ func setupCredentialTestStore(t *testing.T, remoteUser, remotePassword string, s serverMode: serverMode, dbPath: tmpDir, database: dbName, - remote: "origin", + remote: remoteName, } } @@ -359,7 +363,8 @@ func TestCredentialCLIRouting(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { store := setupCredentialTestStore(t, tt.remoteUser, tt.remotePassword, tt.serverMode, tt.setupRemote) - got := store.shouldUseCLIForCredentials(context.Background()) + creds := store.mainRemoteCredentials() + got := store.shouldUseCLIForCredentials(context.Background(), store.remote, creds) if got != tt.wantCLI { t.Errorf("shouldUseCLIForCredentials() = %v, want %v", got, tt.wantCLI) } @@ -378,7 +383,7 @@ func TestCredentialCLIRoutingExternalServer(t *testing.T) { database: "testdb", remote: "origin", } - if store.shouldUseCLIForCredentials(context.Background()) { + if store.shouldUseCLIForCredentials(context.Background(), store.remote, store.mainRemoteCredentials()) { t.Error("expected false for external server mode (no CLI remote in CLIDir)") } } @@ -395,7 +400,7 @@ func TestCredentialCLIRoutingNoRemote(t *testing.T) { database: "nodb", remote: "origin", } - if store.shouldUseCLIForCredentials(context.Background()) { + if store.shouldUseCLIForCredentials(context.Background(), store.remote, store.mainRemoteCredentials()) { t.Error("expected false when CLI remote does not exist") } } @@ -528,38 +533,160 @@ func TestCloudAuthCLIRouting(t *testing.T) { } tests := []struct { - name string - serverMode bool - setupRemote bool - envKey string // env var to set (empty = none) - envValue string - wantCLI bool + name string + remoteURL string // URL scheme determines which env vars are relevant + envKey string // env var to set (empty = none) + envValue string + wantCLI bool }{ - // Positive: cloud env + server mode + remote configured → CLI - {"azure storage account", true, true, "AZURE_STORAGE_ACCOUNT", "myaccount", true}, - {"azure storage key", true, true, "AZURE_STORAGE_KEY", "mykey", true}, - {"aws access key", true, true, "AWS_ACCESS_KEY_ID", "AKID", true}, - {"aws secret key", true, true, "AWS_SECRET_ACCESS_KEY", "secret", true}, - {"google creds", true, true, "GOOGLE_APPLICATION_CREDENTIALS", "/path/to/creds.json", true}, - {"gcs creds file", true, true, "GCS_CREDENTIALS_FILE", "/path/to/creds.json", true}, - {"oci var", true, true, "OCI_TENANCY", "ocid1.tenancy", true}, - {"dolt remote user", true, true, "DOLT_REMOTE_USER", "admin", true}, - // Negative: missing conditions → SQL fallback - {"no cloud env", true, true, "", "", false}, - {"embedded mode", false, true, "AZURE_STORAGE_ACCOUNT", "myaccount", false}, - {"no CLI remote", true, false, "AZURE_STORAGE_ACCOUNT", "myaccount", false}, + // Per-scheme positive: env var matches remote's scheme → CLI routing + {"azure env + az:// remote", "az://account.blob.core.windows.net/container", "AZURE_STORAGE_ACCOUNT", "myaccount", true}, + {"azure key + az:// remote", "az://account.blob.core.windows.net/container", "AZURE_STORAGE_KEY", "mykey", true}, + {"aws env + s3:// remote", "s3://my-bucket/path", "AWS_ACCESS_KEY_ID", "AKID", true}, + {"aws secret + s3:// remote", "s3://my-bucket/path", "AWS_SECRET_ACCESS_KEY", "secret", true}, + {"google creds + gs:// remote", "gs://my-bucket/path", "GOOGLE_APPLICATION_CREDENTIALS", "/path/to/creds.json", true}, + {"gcs creds + gs:// remote", "gs://my-bucket/path", "GCS_CREDENTIALS_FILE", "/path/to/creds.json", true}, + {"oci env + oci:// remote", "oci://my-namespace/my-bucket/path", "OCI_TENANCY", "ocid1.tenancy", true}, + {"dolt env + dolthub:// remote", "dolthub://org/beads", "DOLT_REMOTE_USER", "admin", true}, + {"dolt env + https:// remote", "https://example.com/repo", "DOLT_REMOTE_USER", "admin", true}, + {"dolt env + http:// remote", "http://example.com/repo", "DOLT_REMOTE_USER", "admin", true}, + + // Per-scheme negative: env var does NOT match remote's scheme → SQL fallback + {"azure env + dolthub:// remote", "dolthub://org/beads", "AZURE_STORAGE_ACCOUNT", "myaccount", false}, + {"azure env + https:// remote", "https://example.com/repo", "AZURE_STORAGE_ACCOUNT", "myaccount", false}, + {"azure env + s3:// remote", "s3://my-bucket/path", "AZURE_STORAGE_ACCOUNT", "myaccount", false}, + {"aws env + az:// remote", "az://account.blob.core.windows.net/container", "AWS_ACCESS_KEY_ID", "AKID", false}, + {"dolt env + az:// remote", "az://account.blob.core.windows.net/container", "DOLT_REMOTE_USER", "admin", false}, + + // Structural negative: missing conditions → SQL fallback + {"no cloud env", "az://account.blob.core.windows.net/container", "", "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Use a clean store (no remoteUser/remotePassword — cloud auth uses env vars) - store := setupCredentialTestStore(t, "", "", tt.serverMode, tt.setupRemote) + store := setupCredentialTestStoreWithURL(t, "", "", true, true, "origin", tt.remoteURL) if tt.envKey != "" { t.Setenv(tt.envKey, tt.envValue) } - got := store.shouldUseCLIForCloudAuth() + got := store.shouldUseCLIForCloudAuth(store.remote) if got != tt.wantCLI { t.Errorf("shouldUseCLIForCloudAuth() = %v, want %v", got, tt.wantCLI) } }) } } + +func TestCloudAuthCLIRoutingStructural(t *testing.T) { + if _, err := exec.LookPath("dolt"); err != nil { + t.Skip("dolt not installed") + } + + t.Run("embedded mode", func(t *testing.T) { + store := setupCredentialTestStoreWithURL(t, "", "", false, true, "origin", "az://account.blob.core.windows.net/container") + t.Setenv("AZURE_STORAGE_ACCOUNT", "myaccount") + if store.shouldUseCLIForCloudAuth(store.remote) { + t.Error("expected false in embedded mode") + } + }) + t.Run("no CLI remote", func(t *testing.T) { + store := setupCredentialTestStoreWithURL(t, "", "", true, false, "origin", "az://account.blob.core.windows.net/container") + t.Setenv("AZURE_STORAGE_ACCOUNT", "myaccount") + if store.shouldUseCLIForCloudAuth(store.remote) { + t.Error("expected false when CLI remote not configured") + } + }) +} + +// TestPerRemoteCloudAuthHybrid verifies the core use case: a hybrid setup with +// DoltHub (primary) + Azure (backup) remotes. AZURE_STORAGE_ACCOUNT should +// trigger CLI routing ONLY for the Azure remote, not the DoltHub remote. +func TestPerRemoteCloudAuthHybrid(t *testing.T) { + if _, err := exec.LookPath("dolt"); err != nil { + t.Skip("dolt not installed") + } + + tmpDir := t.TempDir() + dbName := "testdb" + dbDir := filepath.Join(tmpDir, dbName) + if err := os.MkdirAll(dbDir, 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command("dolt", "init") + cmd.Dir = dbDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt init: %s: %v", out, err) + } + // Add two remotes: DoltHub primary + Azure backup + cmd = exec.Command("dolt", "remote", "add", "primary", "dolthub://org/beads") + cmd.Dir = dbDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt remote add primary: %s: %v", out, err) + } + cmd = exec.Command("dolt", "remote", "add", "backup", "az://account.blob.core.windows.net/dolt/beads") + cmd.Dir = dbDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt remote add backup: %s: %v", out, err) + } + + store := &DoltStore{ + serverMode: true, + dbPath: tmpDir, + database: dbName, + remote: "primary", + } + + t.Setenv("AZURE_STORAGE_ACCOUNT", "myaccount") + + // Azure remote should get CLI routing (AZURE_STORAGE_ env matches az:// scheme) + if !store.shouldUseCLIForCloudAuth("backup") { + t.Error("expected CLI routing for az:// remote when AZURE_STORAGE_ACCOUNT is set") + } + + // DoltHub remote should NOT get CLI routing (AZURE_STORAGE_ does not match dolthub:// scheme) + if store.shouldUseCLIForCloudAuth("primary") { + t.Error("expected SQL routing for dolthub:// remote when AZURE_STORAGE_ACCOUNT is set — per-remote resolution should prevent misrouting") + } +} + +// TestEnvPrefixesForRemoteURL verifies the scheme-to-env-prefix mapping. +func TestEnvPrefixesForRemoteURL(t *testing.T) { + tests := []struct { + url string + wantNil bool + wantHas string // one prefix we expect to find + }{ + {"az://account.blob.core.windows.net/container", false, "AZURE_STORAGE_"}, + {"s3://my-bucket/path", false, "AWS_"}, + {"gs://my-bucket/path", false, "GOOGLE_"}, + {"oci://namespace/bucket/path", false, "OCI_"}, + {"dolthub://org/repo", false, "DOLT_REMOTE_"}, + {"https://dolthub.com/org/repo", false, "DOLT_REMOTE_"}, + {"http://localhost:8080/repo", false, "DOLT_REMOTE_"}, + {"git+ssh://host/repo", true, ""}, // git protocol — handled elsewhere + {"ssh://host/repo", true, ""}, // git protocol — handled elsewhere + {"file:///path/to/repo", true, ""}, // local filesystem — no cloud auth + {"git@host:repo.git", true, ""}, // SCP-style — handled elsewhere + } + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + got := envPrefixesForRemoteURL(tt.url) + if tt.wantNil && got != nil { + t.Errorf("envPrefixesForRemoteURL(%q) = %v, want nil", tt.url, got) + } + if !tt.wantNil { + if got == nil { + t.Fatalf("envPrefixesForRemoteURL(%q) = nil, want non-nil", tt.url) + } + found := false + for _, p := range got { + if p == tt.wantHas { + found = true + break + } + } + if !found { + t.Errorf("envPrefixesForRemoteURL(%q) = %v, missing %q", tt.url, got, tt.wantHas) + } + } + }) + } +} diff --git a/internal/storage/dolt/git_remote_test.go b/internal/storage/dolt/git_remote_test.go index 29d70fb22..9d7bb71da 100644 --- a/internal/storage/dolt/git_remote_test.go +++ b/internal/storage/dolt/git_remote_test.go @@ -1092,7 +1092,7 @@ func TestGitRemoteExternalServerRouting(t *testing.T) { // SQL sees git+https:// remote in testdb; CLI directory (clientDataDir) has none. // isGitProtocolRemote should return false to route through SQL. - require.False(t, store.isGitProtocolRemote(ctx)) + require.False(t, store.isGitProtocolRemote(ctx, store.remote)) } // TestCredentialCLIRoutingE2E verifies that Push succeeds via CLI subprocess @@ -1218,8 +1218,8 @@ func TestCredentialCLIRoutingE2E(t *testing.T) { t.Cleanup(func() { store.Close() }) // Verify preconditions: not a git-protocol remote, but credentials trigger CLI routing - require.False(t, store.isGitProtocolRemote(ctx), "file:// is not git-protocol") - require.True(t, store.shouldUseCLIForCredentials(ctx), "should route through CLI for credentials") + require.False(t, store.isGitProtocolRemote(ctx, store.remote), "file:// is not git-protocol") + require.True(t, store.shouldUseCLIForCredentials(ctx, store.remote, store.mainRemoteCredentials()), "should route through CLI for credentials") require.True(t, store.serverMode, "store should be in server mode") // 7. Push should succeed via CLI credential routing diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 1b2ae4eb8..f682387ab 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1806,25 +1806,25 @@ func (s *DoltStore) buildBatchCommitMessage(ctx context.Context, actor string) s // server may lack the git credentials, SSH keys, or credential helpers needed for // network I/O to external git hosts. Returns false when the remote exists only on // an externally-managed server's filesystem and not in the local dbPath. -func (s *DoltStore) isGitProtocolRemote(ctx context.Context) bool { +func (s *DoltStore) isGitProtocolRemote(ctx context.Context, remote string) bool { // Check SQL remotes first remotes, err := s.ListRemotes(ctx) if err == nil { for _, r := range remotes { - if r.Name == s.remote { + if r.Name == remote { if !doltutil.IsGitProtocolURL(r.URL) { return false } // Verify remote exists in CLI directory before routing to CLI push/pull. // When the dolt sql-server is externally managed, remotes may exist only // on the server's filesystem, not in the local dbPath. - return s.CLIDir() != "" && doltutil.FindCLIRemote(s.CLIDir(), s.remote) != "" + return s.CLIDir() != "" && doltutil.FindCLIRemote(s.CLIDir(), remote) != "" } } } // Fall back to CLI remotes (covers drift where remote exists only in filesystem) if s.CLIDir() != "" { - if url := doltutil.FindCLIRemote(s.CLIDir(), s.remote); url != "" { + if url := doltutil.FindCLIRemote(s.CLIDir(), remote); url != "" { return doltutil.IsGitProtocolURL(url) } } @@ -1839,18 +1839,28 @@ func (s *DoltStore) mainRemoteCredentials() *remoteCredentials { return &remoteCredentials{username: s.remoteUser, password: s.remotePassword} } +// credentialsForRemote returns credentials only when the target remote is the +// default remote (s.remote). Non-default remotes get nil creds to avoid sending +// the wrong credentials to the wrong host. +func (s *DoltStore) credentialsForRemote(remote string) *remoteCredentials { + if remote == s.remote { + return s.mainRemoteCredentials() + } + return nil +} + // doltCLIPush shells out to `dolt push` from the database directory. // Used for git-protocol remotes where CALL DOLT_PUSH times out through the SQL connection. // If creds is non-nil, credentials are set on the subprocess environment only, // avoiding process-wide env var races with concurrent goroutines. -func (s *DoltStore) doltCLIPush(ctx context.Context, force bool, creds *remoteCredentials) error { +func (s *DoltStore) doltCLIPush(ctx context.Context, remote string, force bool, creds *remoteCredentials) error { ctx, cancel := context.WithTimeout(ctx, cliExecTimeout) defer cancel() args := []string{"push"} if force { args = append(args, "--force") } - args = append(args, s.remote, s.branch) + args = append(args, remote, s.branch) cmd := exec.CommandContext(ctx, "dolt", args...) // #nosec G204 -- fixed command with validated remote/branch cmd.Dir = s.CLIDir() creds.applyToCmd(cmd) @@ -1864,10 +1874,10 @@ func (s *DoltStore) doltCLIPush(ctx context.Context, force bool, creds *remoteCr // doltCLIPull shells out to `dolt pull` from the database directory. // Used for git-protocol remotes where CALL DOLT_PULL times out through the SQL connection. // If creds is non-nil, credentials are set on the subprocess environment only. -func (s *DoltStore) doltCLIPull(ctx context.Context, creds *remoteCredentials) error { +func (s *DoltStore) doltCLIPull(ctx context.Context, remote string, creds *remoteCredentials) error { ctx, cancel := context.WithTimeout(ctx, cliExecTimeout) defer cancel() - cmd := exec.CommandContext(ctx, "dolt", "pull", s.remote, s.branch) // #nosec G204 -- fixed command + cmd := exec.CommandContext(ctx, "dolt", "pull", remote, s.branch) // #nosec G204 -- fixed command cmd.Dir = s.CLIDir() creds.applyToCmd(cmd) out, err := cmd.CombinedOutput() @@ -1882,92 +1892,84 @@ func (s *DoltStore) doltCLIPull(ctx context.Context, creds *remoteCredentials) e // For non-SSH Hosted Dolt (remoteUser set), uses CALL DOLT_PUSH with --user authentication. // For other remotes (DoltHub, S3, GCS, file), uses CALL DOLT_PUSH via SQL. func (s *DoltStore) Push(ctx context.Context) (retErr error) { - ctx, span := doltTracer.Start(ctx, "dolt.push", - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes(append(s.doltSpanAttrs(), - attribute.String("dolt.remote", s.remote), - attribute.String("dolt.branch", s.branch), - )...), - ) - defer func() { endSpan(span, retErr) }() - creds := s.mainRemoteCredentials() - // Git-protocol remotes: use CLI to avoid MySQL connection timeout during transfer. - // Must check before remoteUser — Hosted Dolt SSH remotes have remoteUser set - // but still need CLI to avoid SQL connection timeout. - // Credentials are passed directly to the subprocess via cmd.Env, avoiding - // process-wide env var races with concurrent goroutines. - if s.isGitProtocolRemote(ctx) { - return s.doltCLIPush(ctx, false, creds) - } - // Credential CLI routing: when credentials are set and server is external, - // route through CLI subprocess so credentials reach the dolt process via - // cmd.Env (applyToCmd). The SQL path's withEnvCredentials sets process-wide - // env vars that an external server cannot see. - if s.shouldUseCLIForCredentials(ctx) { - return s.doltCLIPush(ctx, false, creds) - } - // Cloud auth CLI routing: when cloud storage env vars (AZURE_*, AWS_*, - // etc.) are set and we're in server mode, route through CLI so the dolt - // subprocess inherits the current env. The SQL server may not have these - // vars if it was started in a different context (GH#6). - if s.shouldUseCLIForCloudAuth() { - return s.doltCLIPush(ctx, false, creds) - } - if s.remoteUser != "" { - return withEnvCredentials(creds, func() error { - if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH('--user', ?, ?, ?)", s.remoteUser, s.remote, s.branch); err != nil { - return fmt.Errorf("failed to push to %s/%s: %w", s.remote, s.branch, err) - } - return nil - }) - } - if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH(?, ?)", s.remote, s.branch); err != nil { - return fmt.Errorf("failed to push to %s/%s: %w", s.remote, s.branch, err) - } - return nil + return s.pushToRemote(ctx, s.remote, false) } // ForcePush force-pushes commits to the remote, overwriting remote changes. // Use when the remote has uncommitted changes in its working set. // For git-protocol remotes (SSH, git+https://, git://), uses CLI `dolt push --force` to avoid MySQL connection timeouts. func (s *DoltStore) ForcePush(ctx context.Context) (retErr error) { - ctx, span := doltTracer.Start(ctx, "dolt.force_push", + return s.pushToRemote(ctx, s.remote, true) +} + +// PushRemote pushes commits to a named remote. Unlike Push(), which always +// uses the configured default remote (s.remote), PushRemote targets an +// explicit remote name. Credentials are only applied when the target remote +// matches the default remote; otherwise nil creds are used. +func (s *DoltStore) PushRemote(ctx context.Context, remote string, force bool) error { + return s.pushToRemote(ctx, remote, force) +} + +// pushToRemote is the internal implementation for all push operations. +// It routes through CLI or SQL based on the remote's protocol and credentials. +func (s *DoltStore) pushToRemote(ctx context.Context, remote string, force bool) (retErr error) { + spanName := "dolt.push" + if force { + spanName = "dolt.force_push" + } + ctx, span := doltTracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(append(s.doltSpanAttrs(), - attribute.String("dolt.remote", s.remote), + attribute.String("dolt.remote", remote), attribute.String("dolt.branch", s.branch), )...), ) defer func() { endSpan(span, retErr) }() - creds := s.mainRemoteCredentials() + creds := s.credentialsForRemote(remote) // Git-protocol remotes: use CLI to avoid MySQL connection timeout during transfer. // Must check before remoteUser — Hosted Dolt SSH remotes have remoteUser set // but still need CLI to avoid SQL connection timeout. - // Credentials are passed directly to the subprocess via cmd.Env. - if s.isGitProtocolRemote(ctx) { - return s.doltCLIPush(ctx, true, creds) + // Credentials are passed directly to the subprocess via cmd.Env, avoiding + // process-wide env var races with concurrent goroutines. + if s.isGitProtocolRemote(ctx, remote) { + return s.doltCLIPush(ctx, remote, force, creds) } // Credential CLI routing: when credentials are set and server is external, // route through CLI subprocess so credentials reach the dolt process via // cmd.Env (applyToCmd). The SQL path's withEnvCredentials sets process-wide // env vars that an external server cannot see. - if s.shouldUseCLIForCredentials(ctx) { - return s.doltCLIPush(ctx, true, creds) + if s.shouldUseCLIForCredentials(ctx, remote, creds) { + return s.doltCLIPush(ctx, remote, force, creds) } - // Cloud auth CLI routing (GH#6). - if s.shouldUseCLIForCloudAuth() { - return s.doltCLIPush(ctx, true, creds) + // Cloud auth CLI routing: when cloud storage env vars (AZURE_*, AWS_*, + // etc.) are set and we're in server mode, route through CLI so the dolt + // subprocess inherits the current env. The SQL server may not have these + // vars if it was started in a different context (GH#6). + if s.shouldUseCLIForCloudAuth(remote) { + return s.doltCLIPush(ctx, remote, force, creds) } - if s.remoteUser != "" { + if s.remoteUser != "" && remote == s.remote { return withEnvCredentials(creds, func() error { - if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH('--force', '--user', ?, ?, ?)", s.remoteUser, s.remote, s.branch); err != nil { - return fmt.Errorf("failed to force push to %s/%s: %w", s.remote, s.branch, err) + if force { + if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH('--force', '--user', ?, ?, ?)", s.remoteUser, remote, s.branch); err != nil { + return fmt.Errorf("failed to force push to %s/%s: %w", remote, s.branch, err) + } + } else { + if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH('--user', ?, ?, ?)", s.remoteUser, remote, s.branch); err != nil { + return fmt.Errorf("failed to push to %s/%s: %w", remote, s.branch, err) + } } return nil }) } - if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH('--force', ?, ?)", s.remote, s.branch); err != nil { - return fmt.Errorf("failed to force push to %s/%s: %w", s.remote, s.branch, err) + if force { + if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH('--force', ?, ?)", remote, s.branch); err != nil { + return fmt.Errorf("failed to force push to %s/%s: %w", remote, s.branch, err) + } + } else { + if err := s.execWithLongTimeout(ctx, "CALL DOLT_PUSH(?, ?)", remote, s.branch); err != nil { + return fmt.Errorf("failed to push to %s/%s: %w", remote, s.branch, err) + } } return nil } @@ -1981,10 +1983,24 @@ func (s *DoltStore) ForcePush(ctx context.Context) (retErr error) { // stale dolt_auto_push_* rows on multi-machine setups), the conflicts are // automatically resolved using "theirs" strategy (GH#2466). func (s *DoltStore) Pull(ctx context.Context) (retErr error) { + return s.pullFromRemote(ctx, s.remote) +} + +// PullRemote pulls changes from a named remote. Unlike Pull(), which always +// uses the configured default remote (s.remote), PullRemote targets an +// explicit remote name. Credentials are only applied when the target remote +// matches the default remote; otherwise nil creds are used. +func (s *DoltStore) PullRemote(ctx context.Context, remote string) error { + return s.pullFromRemote(ctx, remote) +} + +// pullFromRemote is the internal implementation for all pull operations. +// It routes through CLI or SQL based on the remote's protocol and credentials. +func (s *DoltStore) pullFromRemote(ctx context.Context, remote string) (retErr error) { ctx, span := doltTracer.Start(ctx, "dolt.pull", trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(append(s.doltSpanAttrs(), - attribute.String("dolt.remote", s.remote), + attribute.String("dolt.remote", remote), attribute.String("dolt.branch", s.branch), )...), ) @@ -2003,13 +2019,13 @@ func (s *DoltStore) Pull(ctx context.Context) (retErr error) { } } - creds := s.mainRemoteCredentials() + creds := s.credentialsForRemote(remote) // Git-protocol remotes: use CLI to avoid MySQL connection timeout during transfer. // Must check before remoteUser — Hosted Dolt SSH remotes have remoteUser set // but still need CLI to avoid SQL connection timeout. // Credentials are passed directly to the subprocess via cmd.Env. - if s.isGitProtocolRemote(ctx) { - if err := s.doltCLIPull(ctx, creds); err != nil { + if s.isGitProtocolRemote(ctx, remote) { + if err := s.doltCLIPull(ctx, remote, creds); err != nil { return err } return nil @@ -2017,26 +2033,26 @@ func (s *DoltStore) Pull(ctx context.Context) (retErr error) { // Credential CLI routing: mirrors git-protocol path. // Skips pullWithAutoResolve (consistent with git-protocol Pull — CLI manages its // own connections and conflict handling). - if s.shouldUseCLIForCredentials(ctx) { - if err := s.doltCLIPull(ctx, creds); err != nil { + if s.shouldUseCLIForCredentials(ctx, remote, creds) { + if err := s.doltCLIPull(ctx, remote, creds); err != nil { return err } return nil } // Cloud auth CLI routing (GH#6). - if s.shouldUseCLIForCloudAuth() { - return s.doltCLIPull(ctx, creds) + if s.shouldUseCLIForCloudAuth(remote) { + return s.doltCLIPull(ctx, remote, creds) } - if s.remoteUser != "" { + if s.remoteUser != "" && remote == s.remote { return withEnvCredentials(creds, func() error { - if err := s.pullWithAutoResolve(ctx, "CALL DOLT_PULL('--user', ?, ?, ?)", s.remoteUser, s.remote, s.branch); err != nil { - return fmt.Errorf("failed to pull from %s/%s: %w", s.remote, s.branch, err) + if err := s.pullWithAutoResolve(ctx, "CALL DOLT_PULL('--user', ?, ?, ?)", s.remoteUser, remote, s.branch); err != nil { + return fmt.Errorf("failed to pull from %s/%s: %w", remote, s.branch, err) } return nil }) } - if err := s.pullWithAutoResolve(ctx, "CALL DOLT_PULL(?, ?)", s.remote, s.branch); err != nil { - return fmt.Errorf("failed to pull from %s/%s: %w", s.remote, s.branch, err) + if err := s.pullWithAutoResolve(ctx, "CALL DOLT_PULL(?, ?)", remote, s.branch); err != nil { + return fmt.Errorf("failed to pull from %s/%s: %w", remote, s.branch, err) } return nil } diff --git a/internal/storage/embeddeddolt/version_control.go b/internal/storage/embeddeddolt/version_control.go index d72633bc1..64457726f 100644 --- a/internal/storage/embeddeddolt/version_control.go +++ b/internal/storage/embeddeddolt/version_control.go @@ -231,6 +231,21 @@ func (s *EmbeddedDoltStore) ForcePush(ctx context.Context) error { }) } +func (s *EmbeddedDoltStore) PushRemote(ctx context.Context, remote string, force bool) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if force { + return versioncontrolops.ForcePush(ctx, db, remote, s.branch) + } + return versioncontrolops.Push(ctx, db, remote, s.branch) + }) +} + +func (s *EmbeddedDoltStore) PullRemote(ctx context.Context, remote string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return versioncontrolops.Pull(ctx, db, remote, s.branch) + }) +} + func (s *EmbeddedDoltStore) Fetch(ctx context.Context, peer string) error { return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { return versioncontrolops.Fetch(ctx, db, peer) diff --git a/internal/storage/remote.go b/internal/storage/remote.go index 932269c12..c2db8386c 100644 --- a/internal/storage/remote.go +++ b/internal/storage/remote.go @@ -11,6 +11,8 @@ type RemoteStore interface { Push(ctx context.Context) error Pull(ctx context.Context) error ForcePush(ctx context.Context) error + PushRemote(ctx context.Context, remote string, force bool) error + PullRemote(ctx context.Context, remote string) error Fetch(ctx context.Context, peer string) error PushTo(ctx context.Context, peer string) error PullFrom(ctx context.Context, peer string) ([]Conflict, error)