diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f1809fd9..f6b79d897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -616,6 +616,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd gate add-waiter` survives contention the store gave up replaying.** + It retried only a version miss, so a convoy adder failed when its guarded + write ended in a serialization failure or deadlock the backend stopped + replaying (Dolt/MySQL 1213/1205, Postgres 40001/40P01), or, over HTTP, in a + 503 busy. Those are now retried, but at most twice: each arrives only after + the store or `bd serve` has spent a whole replay budget on the write (up to + 15s), so the 20-write bound a version miss gets could hold an adder for + minutes. Each retry waits the jittered backoff plus a busy answer's + Retry-After, as `bd serve` sent it. The retry is safe because every round + re-reads the gate, deduplicates and writes the whole list guarded by the + version it read: a 503 is not proof that nothing was written (a gateway can + answer one after the commit), and a write that did land is found on the + re-read, so the waiter is never added twice; when the last write allowed is + answered busy, one more read checks for it before the command fails. A + server fault (500) and an unreachable database are still returned at once, + and once the retries run out the error says no write was confirmed and + names the write contention, instead of saying the gate kept changing. - `bd list --title-contains X` works against an HTTP workspace. The client folds it onto the listing's existing `title` parameter (the store compiles both flags to the same case-insensitive substring predicate), so it needs diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 164b7c316..9b2af31f8 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -416,6 +416,7 @@ go_library( "//internal/hooks", "//internal/httpapi", "//internal/httpclient", + "//internal/httpclient/wire", "//internal/jira", "//internal/linear", "//internal/lockfile", @@ -801,6 +802,7 @@ go_test( "formula_help_test.go", "formula_primitives_test.go", "formula_schema_test.go", + "gate_add_waiter_contention_test.go", "gate_check_listing_test.go", "gate_check_proxied_integration_test.go", "gate_discover_test.go", @@ -1348,6 +1350,7 @@ go_test( "//internal/httpapi", "//internal/httpclient", "//internal/httpclient/encode", + "//internal/httpclient/wire", "//internal/linear", "//internal/metrics", "//internal/migration", diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 25fe25e0a..959c6489e 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -15,9 +15,11 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/httpclient/wire" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/issueops" @@ -294,14 +296,43 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio // seconds per read and write), and a wall-clock budget would fail a convoy on // a slow server after only two or three rounds. Each round has a winner, so // the bound carries at least that many simultaneous adders; the jitter -// spreads them further. The waits themselves add up to at most about 6s, and -// a gate rewritten without pause still fails after the last attempt. +// spreads them further. The backoffs themselves add up to at most about 6s, +// and a gate rewritten without pause still fails after the last attempt. +// +// A write lost to contention the store gave up replaying (see +// gateAddWaiterContended) is not one quick round, so it gets a much tighter +// bound of its own, gateAddWaiterContentionRetries. It reaches this loop only +// after the store or bd serve has spent a whole replay budget on it (up to +// 15s, the server-mode Dolt store's and bd serve's), and the wait after a +// busy answer includes the server's Retry-After (5s once bd serve's budget is +// spent). Two retries, like the HTTP client's for a busy read, hold that tail +// to three such writes (three 15s budgets and two 5s waits: about a minute), +// where twenty would take over six minutes. They count against +// gateAddWaiterMaxAttempts too. const ( - gateAddWaiterMaxAttempts = 20 - gateAddWaiterBackoffBase = 10 * time.Millisecond - gateAddWaiterBackoffCap = 400 * time.Millisecond + gateAddWaiterMaxAttempts = 20 + gateAddWaiterContentionRetries = 2 + gateAddWaiterBackoffBase = 10 * time.Millisecond + gateAddWaiterBackoffCap = 400 * time.Millisecond ) +// gateAddWaiterRetryWait is the wait before retry number retry after the +// failed write err: the jittered backoff, plus the server's Retry-After when +// err is a busy answer that carries one. bd serve sends Retry-After on its +// busy answers: 1s when its admission limiter is saturated, and 5s when its +// transaction retry budget is spent, because a convoy back sooner would hold +// slots while it waited, starving reads. It is honored as sent: the wire +// client already bounds the header (wire.Options.MaxRetryAfter). The jitter on +// top keeps adders that were answered busy together from retrying together. +func gateAddWaiterRetryWait(retry int, err error) time.Duration { + wait := gateAddWaiterBackoff(retry) + var problem *wire.ProblemError + if errors.Is(err, wire.ErrBusy) && errors.As(err, &problem) { + wait += problem.RetryAfter + } + return wait +} + // gateAddWaiterBackoff is the wait before retry number retry (1-based): // jittered in [d/2, d) around d = base * 2^(retry-1), capped. func gateAddWaiterBackoff(retry int) time.Duration { @@ -342,10 +373,10 @@ var gateAddWaiterSleep = func(ctx context.Context, d time.Duration) error { // the role stores the list it is given. func addGateWaiter(ctx context.Context, reader issueops.Reader, lifecycle issueops.Lifecycle, gateID, waiter string) (bool, error) { var lastErr error - attempts := 0 - for attempts < gateAddWaiterMaxAttempts { + attempts, contended := 0, 0 + for attempts < gateAddWaiterMaxAttempts && contended <= gateAddWaiterContentionRetries { if attempts > 0 { - if err := gateAddWaiterSleep(ctx, gateAddWaiterBackoff(attempts)); err != nil { + if err := gateAddWaiterSleep(ctx, gateAddWaiterRetryWait(attempts, lastErr)); err != nil { return false, fmt.Errorf("updating gate: %w", err) } } @@ -376,12 +407,84 @@ func addGateWaiter(ctx context.Context, reader issueops.Reader, lifecycle issueo if err == nil { return true, nil } - if !errors.Is(err, issueops.ErrVersionMismatch) { + if !gateAddWaiterLostRound(err) { return false, fmt.Errorf("updating gate: %w", err) } + if gateAddWaiterContended(err) { + contended++ + } lastErr = err } - return false, fmt.Errorf("updating gate: %s changed on every one of %d attempts: %w", gateID, attempts, lastErr) + if errors.Is(lastErr, wire.ErrBusy) && gateAddWaiterLanded(ctx, reader, gateID, waiter) { + return false, nil + } + return false, gateAddWaiterExhausted(gateID, attempts, contended, lastErr) +} + +// gateAddWaiterLanded reports whether waiter is on the gate gateID, read once +// after addGateWaiter's last write was answered busy. Such a write may have +// landed (see gateAddWaiterLostRound), and every earlier round's re-read +// would have found it, but the last write has no round left; so it gets this +// one read, with no wait first. A failed read reports false. +func gateAddWaiterLanded(ctx context.Context, reader issueops.Reader, gateID, waiter string) bool { + details, err := reader.Get(ctx, issueops.GetRequest{ID: gateID}) + return err == nil && slices.Contains(details.Issue.Waiters, waiter) +} + +// gateAddWaiterLostRound reports whether a failed guarded write is a lost +// round the loop retries: a version miss, or a write lost to contention the +// store gave up replaying (gateAddWaiterContended). +// +// Retrying is safe because of how a round is built, NOT because every busy +// answer means nothing was written. bd serve's own busy answers are no-write, +// but a gateway in front of the server can answer 503 after the write +// committed, and the client maps that uncoded 503 to wire.ErrBusy too. Each +// round re-reads the gate, deduplicates, and writes the whole list guarded by +// the row version it read, so a write that did land is found on the re-read +// (the waiter is already there: nothing more is written) and a stale list can +// never overwrite a newer one. So a retry never adds a waiter twice and never +// drops another adder's. +// +// Everything else is returned at once, including the classes a retry would +// also survive but that are not contention: a refusal, a database bd serve +// cannot reach (wire.ErrDBUnavailable), and a server fault such as an +// indeterminate commit (wire.ErrServerFault, a 500). Those are for the caller +// to see, not for a convoy loop to hammer. +func gateAddWaiterLostRound(err error) bool { + return errors.Is(err, issueops.ErrVersionMismatch) || gateAddWaiterContended(err) +} + +// gateAddWaiterContended reports whether a failed guarded write was lost to +// contention the store gave up replaying. On the direct route that is a +// serialization failure or deadlock, which guarantees a rollback (Dolt and +// MySQL report 1213/1205, a SERIALIZABLE store such as Postgres SQLSTATE +// 40001/40P01): the store's transaction envelope replays one until its budget +// is spent and then returns it. Over HTTP it is wire.ErrBusy, which the client +// maps from bd serve's busy answer (a refused admission, or a spent retry +// budget) and from a 503 whose code it does not know or that has none, such as +// a gateway's. A 503 coded db_unavailable or events_watch_saturated maps to its +// own sentinel, not to this one. +// +// Dolt's 1105 "@autocommit transaction rolled back", which the direct route's +// store also replays, is deliberately not admitted: Dolt raises it for +// conflicts that a merge inside the transaction left behind, while a write +// lost to a concurrent writer, the convoy's case, comes back as 1213. +func gateAddWaiterContended(err error) bool { + return uow.IsSerializationError(err) || errors.Is(err, wire.ErrBusy) +} + +// gateAddWaiterExhausted is addGateWaiter's error once it stops retrying: +// after attempts writes, contended of them answered busy or lost to +// contention, the last with lastErr. Only when none was contended did the gate +// change under every write, so only then does the error say that; otherwise it +// names the contention, so that the operator of a busy server is not sent +// looking for a concurrent writer. It says no write was confirmed, not that +// none landed, because a busy answer does not prove that nothing was written. +func gateAddWaiterExhausted(gateID string, attempts, contended int, lastErr error) error { + if contended == 0 { + return fmt.Errorf("updating gate: %s changed on every one of %d attempts: %w", gateID, attempts, lastErr) + } + return fmt.Errorf("updating gate: no write to %s was confirmed in %d attempts (%d answered busy or lost to write contention): %w", gateID, attempts, contended, lastErr) } // renderGateWaiterAlready and renderGateWaiterAdded are shared by the direct diff --git a/cmd/bd/gate_add_waiter_contention_test.go b/cmd/bd/gate_add_waiter_contention_test.go new file mode 100644 index 000000000..0113b46c8 --- /dev/null +++ b/cmd/bd/gate_add_waiter_contention_test.go @@ -0,0 +1,278 @@ +package main + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "testing" + "time" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// sqlStateError is a driver error carrying a SQLSTATE, the shape pgconn.PgError +// and every other driver error uow.IsSerializationError reads. +type sqlStateError struct{ state string } + +func (e *sqlStateError) Error() string { + return "ERROR: could not serialize access (SQLSTATE " + e.state + ")" +} +func (e *sqlStateError) SQLState() string { return e.state } + +// contendedGateRoles fails the first len(errs) updates with errs, then +// behaves as fakeGateRoles. A failed update writes nothing, except failed +// update number landed (1-based): it lands first and is answered with its +// error anyway, as a gateway's 503 can answer a write the server committed. +// Once every one of errs is spent, a read answers readErr if it is set. +type contendedGateRoles struct { + *fakeGateRoles + errs []error + landed int + failed int + readErr error +} + +func (c *contendedGateRoles) Get(ctx context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + if c.readErr != nil && len(c.errs) == 0 { + return nil, c.readErr + } + return c.fakeGateRoles.Get(ctx, req) +} + +func (c *contendedGateRoles) Update(ctx context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + if len(c.errs) > 0 { + err := c.errs[0] + c.errs = c.errs[1:] + c.failed++ + if c.failed == c.landed { + if _, applyErr := c.fakeGateRoles.Update(ctx, req); applyErr != nil { + return issueops.UpdateResult{}, applyErr + } + return issueops.UpdateResult{}, err + } + c.updates = append(c.updates, req) + return issueops.UpdateResult{}, err + } + return c.fakeGateRoles.Update(ctx, req) +} + +// TestAddGateWaiterRetriesTransientContention: contention the store gave up +// replaying is one more lost round, not a failure, but a costly one. The +// store's transaction envelope replays a serialization failure or deadlock +// (Dolt and MySQL 1213/1205, SQLSTATE 40001/40P01) until its budget is spent +// and then returns it; over HTTP bd serve answers that spent budget, or a +// refused admission, as 503 busy (wire.ErrBusy). The loop retries them under +// the same backoff as a version miss, but only gateAddWaiterContentionRetries +// times, because each such write has already spent a whole replay budget. +// What makes a retry safe is the round itself (re-read, deduplicate, +// version-guarded whole-list write), which the busy-after-commit case pins: a +// 503 is not proof that nothing was written. +func TestAddGateWaiterRetriesTransientContention(t *testing.T) { + setGateTestActor(t) + origSleep := gateAddWaiterSleep + t.Cleanup(func() { gateAddWaiterSleep = origSleep }) + var sleeps []time.Duration + gateAddWaiterSleep = func(_ context.Context, d time.Duration) error { + sleeps = append(sleeps, d) + return nil + } + gate := func() *types.Issue { + return &types.Issue{ID: "bd-g", IssueType: "gate", Waiters: []string{"a"}, RowVersion: 7} + } + doltSerialization := fmt.Errorf("update: %w", &mysql.MySQLError{Number: 1213, Message: "serialization failure: this transaction conflicts with a committed transaction from another client, try restarting transaction"}) + + for _, tt := range []struct { + name string + err error + }{ + {"serialization failure", fmt.Errorf("update: %w", &sqlStateError{state: "40001"})}, + {"deadlock", fmt.Errorf("update: %w", &sqlStateError{state: "40P01"})}, + {"Dolt serialization failure (1213)", doltSerialization}, + {"lock wait timeout (1205)", fmt.Errorf("update: %w", &mysql.MySQLError{Number: 1205, Message: "Lock wait timeout exceeded; try restarting transaction"})}, + {"bd serve busy", fmt.Errorf("updateIssue: %w", wire.ErrBusy)}, + } { + t.Run(tt.name, func(t *testing.T) { + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{tt.err, tt.err}} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added after two contended rounds", added, err) + } + if len(roles.updates) != 3 || len(sleeps) != 2 { + t.Fatalf("updates = %d, sleeps = %d; want 3 writes with a backoff before each retry", len(roles.updates), len(sleeps)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b]", got) + } + }) + } + + for _, tt := range []struct { + name string + err error + }{ + {"bd serve busy", wire.ErrBusy}, + {"Dolt serialization failure", doltSerialization}, + } { + t.Run(tt.name+" gets only the contention retries", func(t *testing.T) { + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: slices.Repeat([]error{tt.err}, gateAddWaiterContentionRetries+2)} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, tt.err) { + t.Fatalf("addGateWaiter error = %v, want the last contended write's %v", err, tt.err) + } + if want := gateAddWaiterContentionRetries + 1; len(roles.updates) != want { + t.Fatalf("updates = %d, want %d: each contended write already spent a whole replay budget", len(roles.updates), want) + } + if len(sleeps) != gateAddWaiterContentionRetries { + t.Errorf("sleeps = %v, want one per retry and none after the last contended write", sleeps) + } + }) + } + + t.Run("version misses do not spend the contention retries", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{ + wire.ErrBusy, wire.ErrBusy, issueops.ErrVersionMismatch, issueops.ErrVersionMismatch, + }} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added || len(roles.updates) != 5 { + t.Fatalf("addGateWaiter = %v, %v after %d updates; want added on the fifth write", added, err, len(roles.updates)) + } + }) + + for _, tt := range []struct { + name string + errs []error + want string + is error + }{ + { + "a gate that changed under every write", + slices.Repeat([]error{issueops.ErrVersionMismatch}, gateAddWaiterMaxAttempts), + "updating gate: bd-g changed on every one of 20 attempts: ", + issueops.ErrVersionMismatch, + }, + { + "writes lost to contention", + slices.Repeat([]error{wire.ErrBusy}, gateAddWaiterContentionRetries+1), + "updating gate: no write to bd-g was confirmed in 3 attempts (3 answered busy or lost to write contention): ", + wire.ErrBusy, + }, + { + "one contended write among version misses", + append([]error{wire.ErrBusy}, slices.Repeat([]error{issueops.ErrVersionMismatch}, gateAddWaiterMaxAttempts-1)...), + "updating gate: no write to bd-g was confirmed in 20 attempts (1 answered busy or lost to write contention): ", + issueops.ErrVersionMismatch, + }, + } { + t.Run("the exhaustion error after "+tt.name, func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: tt.errs} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, tt.is) { + t.Fatalf("addGateWaiter error = %v, want it to wrap %v", err, tt.is) + } + if got := err.Error(); got != tt.want+tt.is.Error() { + t.Errorf("addGateWaiter error = %q, want %q", got, tt.want+tt.is.Error()) + } + }) + } + + t.Run("a busy answer to a write that landed adds the waiter once", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{wire.ErrBusy}, landed: 1} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil { + t.Fatalf("addGateWaiter error = %v, want the retry's re-read to find the landed waiter", err) + } + if added { + t.Errorf("addGateWaiter reported a second add; the re-read must find b already registered") + } + if len(roles.updates) != 1 { + t.Errorf("updates = %d, want only the write that landed", len(roles.updates)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b] with b exactly once", got) + } + }) + + for _, tt := range []struct { + name string + errs []error + }{ + {"the contention bound", slices.Repeat([]error{wire.ErrBusy}, gateAddWaiterContentionRetries+1)}, + {"the attempt bound", append(slices.Repeat([]error{issueops.ErrVersionMismatch}, gateAddWaiterMaxAttempts-1), wire.ErrBusy)}, + } { + t.Run("a busy answer to the last write, at "+tt.name+", that landed is found by one more read", func(t *testing.T) { + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: tt.errs, landed: len(tt.errs)} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || added { + t.Fatalf("addGateWaiter = %v, %v; want no error and no second add once the last read finds b", added, err) + } + if len(roles.updates) != len(tt.errs) || len(sleeps) != len(tt.errs)-1 { + t.Fatalf("updates = %d, sleeps = %d; want %d writes and no wait before the last read", len(roles.updates), len(sleeps), len(tt.errs)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b] with b exactly once", got) + } + }) + } + + t.Run("a failed read after the last busy write confirms nothing", func(t *testing.T) { + errs := slices.Repeat([]error{wire.ErrBusy}, gateAddWaiterContentionRetries+1) + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: errs, landed: len(errs), readErr: fmt.Errorf("getIssue: %w", wire.ErrBusy)} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + want := "updating gate: no write to bd-g was confirmed in 3 attempts (3 answered busy or lost to write contention): " + wire.ErrBusy.Error() + if err == nil || err.Error() != want { + t.Fatalf("addGateWaiter error = %v, want %q", err, want) + } + }) + + for _, tt := range []struct { + name string + err error + }{ + {"a server fault (500, an indeterminate commit)", fmt.Errorf("updateIssue: %w", wire.ErrServerFault)}, + {"an unreachable database", fmt.Errorf("updateIssue: %w", wire.ErrDBUnavailable)}, + } { + t.Run(tt.name+" is not retried", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{tt.err}} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, tt.err) || len(roles.updates) != 1 { + t.Fatalf("addGateWaiter = %v after %d updates, want %v after one write", err, len(roles.updates), tt.err) + } + }) + } + + t.Run("a busy answer's Retry-After is waited out whole, plus the jitter", func(t *testing.T) { + busy := &wire.ProblemError{Status: 503, Code: "busy", RetryAfter: 5 * time.Second, Err: wire.ErrBusy} + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{fmt.Errorf("updateIssue: %w", busy), wire.ErrBusy}} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added after two busy rounds", added, err) + } + if len(sleeps) != 2 { + t.Fatalf("sleeps = %v, want one per retry", sleeps) + } + if lo, hi := busy.RetryAfter+gateAddWaiterBackoffBase/2, busy.RetryAfter+gateAddWaiterBackoffBase; sleeps[0] < lo || sleeps[0] >= hi { + t.Errorf("wait after Retry-After 5s = %s, want the whole 5s plus the first backoff, in [%s, %s)", sleeps[0], lo, hi) + } + if lo, hi := gateAddWaiterBackoffBase, 2*gateAddWaiterBackoffBase; sleeps[1] < lo || sleeps[1] >= hi { + t.Errorf("wait after a busy answer with no Retry-After = %s, want the plain second backoff, in [%s, %s)", sleeps[1], lo, hi) + } + }) + + t.Run("another SQLSTATE is not retried", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{&sqlStateError{state: "23505"}}} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err == nil || len(roles.updates) != 1 { + t.Fatalf("addGateWaiter = %v after %d updates, want one failed write", err, len(roles.updates)) + } + }) +}