From 5d2a40dfc8c8a00a93490f0892d0adb0d912c71d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coffeegoddd=E2=98=95=EF=B8=8F=E2=9C=A8?= Date: Thu, 26 Mar 2026 14:25:20 -0700 Subject: [PATCH] /internal/storage/dolt: withRetryTx --- internal/storage/dolt/compact.go | 2 +- internal/storage/dolt/config.go | 6 +++--- internal/storage/dolt/dependencies.go | 2 +- internal/storage/dolt/events.go | 2 +- internal/storage/dolt/issues.go | 12 ++++++------ internal/storage/dolt/labels.go | 4 ++-- internal/storage/dolt/queries.go | 2 +- internal/storage/dolt/rename.go | 4 ++-- internal/storage/dolt/retry_test.go | 18 ++++++++++++++++++ internal/storage/dolt/store.go | 21 ++++++++++++++++++++- 10 files changed, 55 insertions(+), 18 deletions(-) diff --git a/internal/storage/dolt/compact.go b/internal/storage/dolt/compact.go index fabf40ebc..e6a0006ee 100644 --- a/internal/storage/dolt/compact.go +++ b/internal/storage/dolt/compact.go @@ -48,7 +48,7 @@ func (s *DoltStore) CheckEligibility(ctx context.Context, issueID string, tier i // ApplyCompaction records a compaction result in the database. func (s *DoltStore) ApplyCompaction(ctx context.Context, issueID string, tier int, originalSize int, _ int, commitHash string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.ApplyCompactionInTx(ctx, tx, issueID, tier, originalSize, commitHash) }) } diff --git a/internal/storage/dolt/config.go b/internal/storage/dolt/config.go index 52d971229..dd4b385a3 100644 --- a/internal/storage/dolt/config.go +++ b/internal/storage/dolt/config.go @@ -14,7 +14,7 @@ import ( // SetConfig sets a configuration value func (s *DoltStore) SetConfig(ctx context.Context, key, value string) error { - if err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.SetConfigInTx(ctx, tx, key, value) }); err != nil { return err @@ -70,14 +70,14 @@ func (s *DoltStore) GetAllConfig(ctx context.Context) (map[string]string, error) // DeleteConfig removes a configuration value func (s *DoltStore) DeleteConfig(ctx context.Context, key string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.DeleteConfigInTx(ctx, tx, key) }) } // SetMetadata sets a metadata value func (s *DoltStore) SetMetadata(ctx context.Context, key, value string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.SetMetadataInTx(ctx, tx, key, value) }) } diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index 209c54f41..347d0dc4e 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -35,7 +35,7 @@ func (s *DoltStore) AddDependency(ctx context.Context, dep *types.Dependency, ac } } - if err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { opts := issueops.AddDependencyOpts{ SourceTable: "issues", TargetTable: targetTable, diff --git a/internal/storage/dolt/events.go b/internal/storage/dolt/events.go index 5c4de9dc4..b624080e4 100644 --- a/internal/storage/dolt/events.go +++ b/internal/storage/dolt/events.go @@ -56,7 +56,7 @@ func (s *DoltStore) AddIssueComment(ctx context.Context, issueID, author, text s // This prevents comment timestamp drift across import/export cycles. func (s *DoltStore) ImportIssueComment(ctx context.Context, issueID, author, text string, createdAt time.Time) (*types.Comment, error) { var result *types.Comment - err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { var err error result, err = issueops.ImportIssueCommentInTx(ctx, tx, issueID, author, text, createdAt) return err diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index b0a959417..5a89b455d 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -27,7 +27,7 @@ func (s *DoltStore) CreateIssue(ctx context.Context, issue *types.Issue, actor s issue.Ephemeral = true // infra types get marked ephemeral (legacy behavior) } - if err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { // SkipPrefixValidation matches legacy behavior: single-issue path does // not validate prefixes for explicit IDs. bc, err := issueops.NewBatchContext(ctx, tx, storage.BatchCreateOptions{ @@ -73,7 +73,7 @@ func (s *DoltStore) CreateIssuesWithFullOptions(ctx context.Context, issues []*t if !issue.NoHistory { issue.Ephemeral = true } - if err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { bc, err := issueops.NewBatchContext(ctx, tx, opts) if err != nil { return err @@ -86,7 +86,7 @@ func (s *DoltStore) CreateIssuesWithFullOptions(ctx context.Context, issues []*t return nil } - if err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.CreateIssuesInTx(ctx, tx, issues, actor, opts) }); err != nil { return err @@ -575,7 +575,7 @@ var ( // Returns the number of issues deleted. func (s *DoltStore) DeleteIssuesBySourceRepo(ctx context.Context, sourceRepo string) (int, error) { var count int - err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { var err error count, err = issueops.DeleteIssuesBySourceRepoInTx(ctx, tx, sourceRepo) return err @@ -588,7 +588,7 @@ func (s *DoltStore) DeleteIssuesBySourceRepo(ctx context.Context, sourceRepo str // ClearRepoMtime removes the mtime cache entry for a repository. func (s *DoltStore) ClearRepoMtime(ctx context.Context, repoPath string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.ClearRepoMtimeInTx(ctx, tx, repoPath) }) } @@ -607,7 +607,7 @@ func (s *DoltStore) GetRepoMtime(ctx context.Context, repoPath string) (int64, e // SetRepoMtime updates the mtime cache for a repository's data file. func (s *DoltStore) SetRepoMtime(ctx context.Context, repoPath, jsonlPath string, mtimeNs int64) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.SetRepoMtimeInTx(ctx, tx, repoPath, jsonlPath, mtimeNs) }) } diff --git a/internal/storage/dolt/labels.go b/internal/storage/dolt/labels.go index d48031a8b..b6ee76b6a 100644 --- a/internal/storage/dolt/labels.go +++ b/internal/storage/dolt/labels.go @@ -10,7 +10,7 @@ import ( // AddLabel adds a label to an issue func (s *DoltStore) AddLabel(ctx context.Context, issueID, label, actor string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.AddLabelInTx(ctx, tx, "", "", issueID, label, actor) }) } @@ -18,7 +18,7 @@ func (s *DoltStore) AddLabel(ctx context.Context, issueID, label, actor string) // RemoveLabel removes a label from an issue. // Delegates SQL work to issueops.RemoveLabelInTx which handles wisp routing. func (s *DoltStore) RemoveLabel(ctx context.Context, issueID, label, actor string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.RemoveLabelInTx(ctx, tx, "", "", issueID, label, actor) }) } diff --git a/internal/storage/dolt/queries.go b/internal/storage/dolt/queries.go index d3775fa25..12e6a8639 100644 --- a/internal/storage/dolt/queries.go +++ b/internal/storage/dolt/queries.go @@ -542,7 +542,7 @@ func (s *DoltStore) GetMoleculeLastActivity(ctx context.Context, moleculeID stri // Delegates SQL work to issueops.GetNextChildIDTx. func (s *DoltStore) GetNextChildID(ctx context.Context, parentID string) (string, error) { var childID string - err := s.withWriteTx(ctx, func(tx *sql.Tx) error { + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { var err error childID, err = issueops.GetNextChildIDTx(ctx, tx, parentID) return err diff --git a/internal/storage/dolt/rename.go b/internal/storage/dolt/rename.go index fe25fa72d..ca8b0f84f 100644 --- a/internal/storage/dolt/rename.go +++ b/internal/storage/dolt/rename.go @@ -10,14 +10,14 @@ import ( // UpdateIssueID updates an issue ID and all its references. func (s *DoltStore) UpdateIssueID(ctx context.Context, oldID, newID string, issue *types.Issue, actor string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.UpdateIssueIDInTx(ctx, tx, oldID, newID, issue, actor) }) } // RenameDependencyPrefix updates the prefix in all dependency records func (s *DoltStore) RenameDependencyPrefix(ctx context.Context, oldPrefix, newPrefix string) error { - return s.withWriteTx(ctx, func(tx *sql.Tx) error { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { return issueops.RenameDependencyPrefixInTx(ctx, tx, oldPrefix, newPrefix) }) } diff --git a/internal/storage/dolt/retry_test.go b/internal/storage/dolt/retry_test.go index e4416b622..896f0ed7b 100644 --- a/internal/storage/dolt/retry_test.go +++ b/internal/storage/dolt/retry_test.go @@ -3,7 +3,10 @@ package dolt import ( "context" "errors" + "fmt" "testing" + + mysql "github.com/go-sql-driver/mysql" ) func TestIsRetryableError(t *testing.T) { @@ -87,6 +90,21 @@ func TestIsRetryableError(t *testing.T) { err: errors.New("Error 1105 (HY000): no root value found in session"), expected: true, }, + { + name: "MySQL deadlock (error 1213) - retryable", + err: &mysql.MySQLError{Number: 1213, Message: "Deadlock found when trying to get lock"}, + expected: true, + }, + { + name: "MySQL lock wait timeout (error 1205) - retryable", + err: &mysql.MySQLError{Number: 1205, Message: "Lock wait timeout exceeded"}, + expected: true, + }, + { + name: "wrapped MySQL deadlock - retryable", + err: fmt.Errorf("exec failed: %w", &mysql.MySQLError{Number: 1213, Message: "Deadlock found"}), + expected: true, + }, { name: "syntax error - not retryable", err: errors.New("Error 1064: You have an error in your SQL syntax"), diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index c656e6994..2b8219500 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -255,12 +255,22 @@ func newServerRetryBackoff() backoff.BackOff { return bo } -// isRetryableError returns true if the error is a transient connection error +// isRetryableError returns true if the error is a transient error // that should be retried in server mode. func isRetryableError(err error) bool { if err == nil { return false } + // MySQL server-level errors (typed) + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) { + switch mysqlErr.Number { + case 1213: // ER_LOCK_DEADLOCK: "Deadlock found when trying to get lock" + return true + case 1205: // ER_LOCK_WAIT_TIMEOUT: "Lock wait timeout exceeded" + return true + } + } errStr := strings.ToLower(err.Error()) // MySQL driver transient errors if strings.Contains(errStr, "driver: bad connection") { @@ -471,6 +481,15 @@ func (s *DoltStore) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) e return fn(tx) } +// withRetryTx wraps withWriteTx in withRetry so that transient errors +// (bad connection, broken pipe, etc.) cause the entire transaction to +// be retried with exponential backoff and circuit-breaker integration. +func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) error { + return s.withRetry(ctx, func() error { + return s.withWriteTx(ctx, fn) + }) +} + // withWriteTx runs fn inside a transaction, committing on success. // Used for write operations that delegate SQL work to issueops functions. // The caller's fn should NOT call tx.Commit — withWriteTx handles that.