From 773b537c75c3fdf7c38371d5da5cd3d98b615e71 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 15 Oct 2026 23:43:54 +0000 Subject: [PATCH] fix(httpclient): dial a large batchApply under the server's extended budget (#7582) * fix(httpclient): dial a large batchApply under the server's extended budget bd serve runs an issues:batchApply over largeApplyItemThreshold (100) items under DefaultLargeApplyCeiling (5m) on a context detached from the client, but the wire client capped every request at DefaultTimeout (60s). A large apply on a slow or loaded server therefore failed client-side with "Client.Timeout exceeded while awaiting headers" while the server went on to commit it: an error for an apply that landed. wire.Request.Timeout now raises (never lowers) a bounded client's per-request ceiling for one request, through a shallow copy of the *http.Client that shares its Transport and forced CheckRedirect. ApplyBatch sets it to LargeApplyTimeout (6m) once the plan exceeds LargeApplyItemThreshold. The stall-retry deadline uses the same per-request ceiling. Tests: - wire: a slow fake server times out an at-threshold apply under a short ceiling and completes an over-threshold one; httpClientFor only raises a bounded client and keeps its Transport and CheckRedirect. - httpapi: TestTheClientLargeApplyBudgetCoversTheServer pins the redeclared threshold to the server's and holds LargeApplyTimeout above request read + large-slot wait + ceiling + drain grace (5m45s). - httpclient (served tier): a 101-item apply against a BatchApplier delayed DefaultTimeout+5s, dialed through DialWith + DialOptionsForTarget like the product, commits and returns success (71s). With the raise disabled it fails at 60.03s with Client.Timeout. Ported from bd-enterprise (gascity/bd-enterprise#195). Co-Authored-By: Claude Opus 5.5 (1M context) * test(httpclient): pin the raised stall-retry deadline; document the 6m large-apply ceiling Review fixups for #7582. - wire: TestARaisedRequestTimeoutAlsoBoundsTheStallRetry pins the one OSS-only hunk of the port, the stall-retry deadline read through httpClientFor(r). Reverting that line to c.hc.Timeout left the whole wire package green; the new case fails on it, because the retry then runs out at the client's own ceiling instead of the request's raised one. - backend/http: Options.HTTPClient said the client is used as given except for redirect handling. A batch apply over 100 items now raises a Timeout shorter than 6 minutes for that request, so the public doc says so. - README and CHANGELOG: the HTTP backend's write bound is no longer 60s for every request. A batch apply over 100 items gets 6 minutes, and a server whose --large-apply-ceiling is raised past about 5m15s can still outlast it. - engdocs/SERVE_RUNBOOK.md: the same limit, next to the stop-grace sizing an operator reads before raising --large-apply-ceiling. - served_batch_apply_test: the race-build comment blamed the client's 60s timeout for shrinking the 1000-item apply. A plan that size now gets wire.LargeApplyTimeout; it stays shrunk because the embedded tier measured it at 903-945s under -race. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 ++ README.md | 13 +- backend/http/http.go | 12 +- engdocs/SERVE_RUNBOOK.md | 9 ++ internal/httpapi/BUILD.bazel | 2 + .../httpapi/large_apply_client_budget_test.go | 42 ++++++ internal/httpclient/BUILD.bazel | 1 + .../httpclient/served_batch_apply_test.go | 12 +- internal/httpclient/served_harness_test.go | 4 + .../served_large_apply_timeout_test.go | 105 ++++++++++++++ internal/httpclient/wire/BUILD.bazel | 1 + internal/httpclient/wire/client.go | 77 ++++++++-- .../wire/large_apply_timeout_test.go | 135 ++++++++++++++++++ internal/httpclient/wire/stall_retry_test.go | 53 +++++++ internal/httpclient/wire/writes.go | 6 +- 15 files changed, 466 insertions(+), 21 deletions(-) create mode 100644 internal/httpapi/large_apply_client_budget_test.go create mode 100644 internal/httpclient/served_large_apply_timeout_test.go create mode 100644 internal/httpclient/wire/large_apply_timeout_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad19a96a..715f28594 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -550,6 +550,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 naming the server's host, because it may or may not have been applied, while a dial or TLS timeout stays a plain connection error. The batch get, a read on POST, is now also retried on a 503 the way a GET is. +- **A large batch apply over HTTP no longer fails at 60s while the server + commits it.** `bd serve` runs an `issues:batchApply` of more than 100 items + (a large `bd create --graph` plan, say) under its `--large-apply-ceiling` + (5 minutes by default), on a context detached from the client, but the HTTP + backend client gave every request 60s. A large apply that took longer + failed client-side with a timeout while the server went on to commit it. + Such a request is now dialed with 6 minutes, which covers the server's + default ceiling plus what it may spend before (reading the request, waiting + for the one-wide large-apply slot) and after (writing the answer back). + Every other request keeps 60s. An `*http.Client` passed in through + `backend/http`'s `Options.HTTPClient` is raised the same way for that one + request when its `Timeout` is shorter than 6 minutes; one with no `Timeout` + stays unbounded. A server whose `--large-apply-ceiling` is raised past about + 5m15s can still outlast the client: it does not advertise its ceiling, so + the client cannot learn it. - **An update of a template is now refused for every caller, not only `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item diff --git a/README.md b/README.md index 570bdc01c..5760ce630 100644 --- a/README.md +++ b/README.md @@ -269,10 +269,15 @@ which a Windows reset (`WSAECONNRESET`) is not. A write gets no header timer (the server answers a write only after it commits, so a large one can be slow and healthy) and is never retried; if the 60s bound expires after it was sent but before its headers, it fails with an error naming the -server's host, because it may or may not have been applied. A dial or TLS -timeout stays a plain connection error, since the server never saw the -request. `off` turns off only the header timer: a reset read is still -retried, and a write's timeout is still reported as above. +server's host, because it may or may not have been applied. A batch apply of +more than 100 items (a large `bd create --graph` plan, say) gets 6 minutes +instead of 60s, because `bd serve` gives such a request its +`--large-apply-ceiling` (5 minutes by default) and finishes it even after the +client has given up. A server whose ceiling is raised past about 5m15s can +still outlast the client. A dial or TLS timeout stays a plain connection +error, since the server never saw the request. `off` turns off only the +header timer: a reset read is still retried, and a write's timeout is still +reported as above. `bd search`, `bd show --current` and `bd recompute-blocked` work over HTTP too, each in its own way. `bd search` asks the server's listing with its text (`q`, behind the `issues.list.search` capability) and diff --git a/backend/http/http.go b/backend/http/http.go index 5f6e085ca..3d2b5f734 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -90,9 +90,15 @@ type Options struct { UserAgent string // HTTPClient replaces the transport — a tuned timeout, a proxy, a custom // TLS config, an httptest server in a test. Nil builds one. The client is - // used as given except for redirect handling, which the underlying wire - // client always forces off: a followed 30x would replay the Authorization - // header at whatever host the Location named. + // used as given except for two things the underlying wire client always + // does. It forces redirect handling off: a followed 30x would replay the + // Authorization header at whatever host the Location named. And for a + // batch apply of more than 100 items it raises a Timeout shorter than 6 + // minutes to 6 minutes, for that request alone: bd serve gives such a + // request an extended budget and commits it even after the client has + // given up, so a shorter Timeout reports a failure for a write that + // lands. A client with no Timeout stays unbounded; a call's context still + // bounds it. HTTPClient *http.Client // RequireCredential makes the registered backend's OpenWith seam refuse a // registered open with no OpenOptions.Credential, rather than silently diff --git a/engdocs/SERVE_RUNBOOK.md b/engdocs/SERVE_RUNBOOK.md index a6a04123f..fb4f14b99 100644 --- a/engdocs/SERVE_RUNBOOK.md +++ b/engdocs/SERVE_RUNBOOK.md @@ -337,6 +337,15 @@ waiting for, which can still sever an in-flight large apply the drain would otherwise have waited out. Size the grace period to the ceiling plus those 5 seconds, not to the ordinary 20s drain. +**Raising `--large-apply-ceiling` past about 5m15s also outruns bd's own HTTP +client.** It waits 6 minutes for a batch apply over 100 items: the default +ceiling plus what the server may spend around it (the 30s request read, the +10s wait for the large-apply slot, 5s to write the answer back). The server +does not advertise its ceiling, so a client cannot wait longer for a server +that grants more, and an apply that runs past the client's 6 minutes is +reported to the caller as a timeout with an unknown outcome while the server +goes on to commit it. + Journal streams are the exception, and they have to be: a held-open response would otherwise sit through the whole budget and then be killed anyway. They get an explicit close signal when the drain starts and end on their own within a diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index efe2f12b3..54d0f3f96 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -89,6 +89,7 @@ go_test( "events_test.go", "events_watch_test.go", "get_issue_test.go", + "large_apply_client_budget_test.go", "large_batch_apply_test.go", "list_comments_omitted_test.go", "list_ids_test.go", @@ -127,6 +128,7 @@ go_test( "//internal/eventsjournal", "//internal/httpapi/apigen", "//internal/httpapi/spec", + "//internal/httpclient/wire", "//internal/storage", "//internal/storage/domain", "//internal/storage/uow", diff --git a/internal/httpapi/large_apply_client_budget_test.go b/internal/httpapi/large_apply_client_budget_test.go new file mode 100644 index 000000000..1e2cecbb8 --- /dev/null +++ b/internal/httpapi/large_apply_client_budget_test.go @@ -0,0 +1,42 @@ +// Contributed by gascity from bd-enterprise (internal/httpapi/ent_large_apply_client_budget_test.go@1ee63dff5) +// to OSS beads under the MIT license. +package httpapi + +import ( + "testing" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// TestTheClientLargeApplyBudgetCoversTheServer holds the client's redeclared +// large-apply vocabulary (internal/httpclient/wire, which must not import this +// package) to the server's: the client must switch to its extended ceiling at +// exactly the item count where this server switches to its extended budget, +// and that ceiling must outlast everything the server may spend on such a +// request before it answers. Otherwise the client reports a failure for an +// apply the server, running detached from it, goes on to commit. +// +// What the server may spend, as the client's clock sees it: +// - the request read, which readTimeout bounds as a whole and which spans +// both route()'s wait for a general slot (acquire, up to +// semAcquireTimeout) and the body upload the handler then decodes; +// - the wait for the one-wide large-write slot (acquireLargeApply, up to +// semAcquireTimeout); +// - the run itself, DefaultLargeApplyCeiling from admission; +// - drainGrace, the server's own allowance for writing a committed answer +// back. +func TestTheClientLargeApplyBudgetCoversTheServer(t *testing.T) { + if wire.LargeApplyItemThreshold != largeApplyItemThreshold { + t.Errorf("wire.LargeApplyItemThreshold = %d, server largeApplyItemThreshold = %d", + wire.LargeApplyItemThreshold, largeApplyItemThreshold) + } + read := max(readTimeout, semAcquireTimeout) + need := read + semAcquireTimeout + DefaultLargeApplyCeiling + drainGrace + if wire.LargeApplyTimeout < need { + t.Errorf("wire.LargeApplyTimeout = %v; the server may spend up to %v (request read %v + large-write slot wait %v + ceiling %v + drain grace %v)", + wire.LargeApplyTimeout, need, read, semAcquireTimeout, DefaultLargeApplyCeiling, drainGrace) + } + if wire.DefaultTimeout >= wire.LargeApplyTimeout { + t.Errorf("wire.DefaultTimeout = %v is not shorter than wire.LargeApplyTimeout = %v", wire.DefaultTimeout, wire.LargeApplyTimeout) + } +} diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 95ee71185..4401ab1f2 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -152,6 +152,7 @@ go_test( "served_hooks_test.go", "served_issue_operations_test.go", "served_journal_test.go", + "served_large_apply_timeout_test.go", "served_lease_reclaimer_test.go", "served_lifecycle_create_test.go", "served_list_test.go", diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index a168048a7..86b22f1da 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -221,11 +221,13 @@ func TestServedBatchApplyLandsAnIdempotencyRecordWithItsWork(t *testing.T) { func TestServedBatchApplyBoundsTheItemCount(t *testing.T) { fixture := newServedBatchApplyFixture(t, "hba23") if raceEnabled { - // Under -race the full 1000-item apply outlasts the client's 60s - // per-request timeout (wire.DefaultTimeout) on the embedded lane's - // executors, so only the "at bound" half's applied count drops, to - // 150 (the whole case then took 26s there), as the embedded tier's - // own TestBatchApplyContract does; see + // Under -race the full 1000-item apply is far too slow for the + // embedded lane's executors: the embedded tier measured it at + // 903-945s (RunBatchApplyBoundsTheItemCountAtScale's doc), past both + // the server's 5-minute large-apply budget and the client's + // wire.LargeApplyTimeout. So only the "at bound" half's applied count + // drops, to 150 (the whole case then took 26s there), as the embedded + // tier's own TestBatchApplyContract does; see // RunBatchApplyBoundsTheItemCountAtScale for what that leaves pinned. // The refusing half still sends MaxApplyBatchItems+1 over the wire, // and scripts/conformance.sh runs the full size without -race. diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go index 8fe00ca67..4ca99dc27 100644 --- a/internal/httpclient/served_harness_test.go +++ b/internal/httpclient/served_harness_test.go @@ -58,6 +58,9 @@ type servedEnv struct { dataDir string database string prefix string + // base is the in-process server's URL, for a case that dials its own + // client the way the product does rather than through subject. + base *url.URL } const servedDatabase = "httpconf" @@ -185,6 +188,7 @@ func newServedEnvExpecting(t *testing.T, prefix, expectID string, opts ...func(* dataDir: filepath.Join(beadsDir, "embeddeddolt"), database: servedDatabase, prefix: prefix, + base: base, } } diff --git a/internal/httpclient/served_large_apply_timeout_test.go b/internal/httpclient/served_large_apply_timeout_test.go new file mode 100644 index 000000000..3ac922a03 --- /dev/null +++ b/internal/httpclient/served_large_apply_timeout_test.go @@ -0,0 +1,105 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/httpclient/ent_served_large_apply_timeout_test.go@1ee63dff5) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// slowBatchApplier delays the served BatchApplier before delegating, standing +// in for a large apply on a slow or contended server. +type slowBatchApplier struct { + inner issueops.BatchApplier + delay time.Duration +} + +func (s slowBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (issueops.ApplyBatchResult, error) { + select { + case <-time.After(s.delay): + case <-ctx.Done(): + return issueops.ApplyBatchResult{}, ctx.Err() + } + return s.inner.ApplyBatch(ctx, req) +} + +// TestServedLargeApplyOutlivesTheDefaultClientTimeout is the product-path +// regression for a batchApply over the server's large-apply threshold that +// the server takes LONGER than wire.DefaultTimeout to answer. +// The server grants such a request httpapi.DefaultLargeApplyCeiling (5m) on a +// context detached from the client, so it commits regardless; before the fix +// the client gave up at 60s with "Client.Timeout exceeded while awaiting +// headers" and reported a failure for an apply that then landed. +// +// The client is dialed the way the product dials one — DialWith with no +// caller-supplied *http.Client, as OpenWith and backend/http's Open do — so +// it carries the production transport +// and wire.DefaultTimeout rather than the harness's bare wire.New. A ceiling +// the dial path adds (a transport-level header timeout, say) would fail this +// case the way the original one did. +// +// It deliberately spends just over a minute: the point is the PRODUCT client's +// real ceiling, which no test hook shortens. Like every served case it runs +// only in the served tier (newServedEnv skips it unless +// BEADS_TEST_EMBEDDED_DOLT=1). +func TestServedLargeApplyOutlivesTheDefaultClientTimeout(t *testing.T) { + delay := wire.DefaultTimeout + 5*time.Second + env := newServedEnv(t, "hbalt", func(cfg *httpapi.Config) { + if cfg.BatchApplier == nil { + // Failed, not skipped: in the required served tier a skip would + // drop this regression silently. Port the delay to whatever now + // serves batchApply instead. + t.Fatal("the served config binds no BatchApplier role to delay") + } + cfg.BatchApplier = slowBatchApplier{inner: cfg.BatchApplier, delay: delay} + }) + target := Target{BaseURL: env.base} + conn, err := DialWith(target, nil, DialOptionsForTarget(target, DialOptions{})) + if err != nil { + t.Fatalf("DialWith(%s): %v", target, err) + } + applier, err := New(target, conn, nil).BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + + const title = "large apply past the default client timeout" + n := wire.LargeApplyItemThreshold + 1 + items := make([]issueops.ApplyItem, n) + for i := range items { + items[i] = issueops.ApplyItem{Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Issue: &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + }} + } + + start := time.Now() + result, err := applier.ApplyBatch(t.Context(), issueops.ApplyBatchRequest{Actor: "apply-writer", Items: items}) + elapsed := time.Since(start) + if err != nil { + t.Fatalf("ApplyBatch(%d items) with the server taking %v: %v (after %v); a large apply must be dialed "+ + "under wire.LargeApplyTimeout, not wire.DefaultTimeout", n, delay, err, elapsed) + } + if elapsed < wire.DefaultTimeout { + t.Fatalf("ApplyBatch returned after %v, before wire.DefaultTimeout (%v): the delay never engaged, so this "+ + "case proved nothing", elapsed, wire.DefaultTimeout) + } + if len(result.Items) != n { + t.Fatalf("ApplyBatch returned %d item results, want %d", len(result.Items), n) + } + var rows int + if err := env.queryScalar(t.Context(), "SELECT COUNT(*) FROM issues WHERE title = ?", []any{title}, &rows); err != nil { + t.Fatalf("count the applied rows: %v", err) + } + if rows != n { + t.Errorf("%d row(s) landed, want %d", rows, n) + } +} diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel index 53dc2522b..11ff9db08 100644 --- a/internal/httpclient/wire/BUILD.bazel +++ b/internal/httpclient/wire/BUILD.bazel @@ -32,6 +32,7 @@ go_test( "handshake_test.go", "handshake_wire_revision_test.go", "helpers_test.go", + "large_apply_timeout_test.go", "paths_test.go", "problem_test.go", "problem_wire_revision_test.go", diff --git a/internal/httpclient/wire/client.go b/internal/httpclient/wire/client.go index 90c8beb9c..3523acdca 100644 --- a/internal/httpclient/wire/client.go +++ b/internal/httpclient/wire/client.go @@ -50,18 +50,49 @@ const ( DefaultMaxRetryAfter = 30 * time.Second // DefaultTimeout is the per-request ceiling for a client this package built. // A caller's context still wins when it is shorter, and an injected - // *http.Client keeps its own. Exported so a caller that must build its own - // *http.Client — to layer in a custom RoundTripper such as the one - // httpstore.TransportFor returns — can match this default rather than - // silently dropping it. + // *http.Client keeps its own, except that a large apply raises a bounded + // one for that request alone (Request.Timeout, LargeApplyTimeout). + // Exported so a caller that must build its own *http.Client — to layer in + // a custom RoundTripper such as the one httpstore.TransportFor returns — + // can match this default rather than silently dropping it. DefaultTimeout = 60 * time.Second + // LargeApplyTimeout is the per-request ceiling for the one request shape + // bd serve grants MORE than DefaultTimeout: a POST issues:batchApply + // carrying more than LargeApplyItemThreshold items. The server runs such a + // request under its own extended budget — httpapi.DefaultLargeApplyCeiling + // (5m, `bd serve --large-apply-ceiling`), timed from the moment it takes + // the one-wide large-write slot — on a context DETACHED from the client's, + // so a client that gave up at DefaultTimeout would report a failure for an + // apply the server goes on to commit. Before that budget starts the server + // may already have spent its whole request-read timeout (30s, which spans + // its wait for a general slot and the body upload) and its bounded wait + // for the large-write slot (10s); after the commit it still has to write + // the answer back. This value covers all of that at the server's default + // ceiling, and internal/httpapi's + // TestTheClientLargeApplyBudgetCoversTheServer holds it to the server's + // constants. A caller's context still wins when it is shorter, and an + // injected *http.Client with no Timeout at all stays unbounded. An + // operator who raises --large-apply-ceiling past about 5m15s is not + // covered: the server does not advertise its ceiling, so the client + // cannot learn it. + LargeApplyTimeout = 6 * time.Minute ) +// LargeApplyItemThreshold mirrors httpapi's own largeApplyItemThreshold: the +// item count a batchApply request must EXCEED to run under the server's +// extended large-apply budget, and so to be dialed under LargeApplyTimeout +// here. Redeclared rather than imported for ProjectIDHeader's reason; held to +// the server's value by internal/httpapi's +// TestTheClientLargeApplyBudgetCoversTheServer. +const LargeApplyItemThreshold = 100 + // Options tunes the client. The zero value is usable: every field falls back to // the Default* constants above. type Options struct { // HTTPClient is used as given except for CheckRedirect, which is always - // forced (see New). Nil builds one. + // forced (see New), and a positive Timeout, which a request the server + // grants longer (Request.Timeout) raises for that request alone. Nil + // builds one. HTTPClient *http.Client // UserAgent is sent on every request. UserAgent string @@ -242,6 +273,12 @@ type Request struct { // as safe to replay, the way a GET is (see Replayable). Never set it on a // write. Idempotent bool + // Timeout, when positive, RAISES the client's per-request ceiling for this + // one request — never lowers it, and never bounds a client whose + // *http.Client has no Timeout. It exists for the requests the server + // itself grants longer than DefaultTimeout (see LargeApplyTimeout); + // shortening a request is the caller's context's job. + Timeout time.Duration } // Replayable reports whether r may be sent again after an attempt that got no @@ -414,8 +451,8 @@ func (c *Client) roundTrip(ctx context.Context, r Request, u *url.URL, body []by return res, nil } var deadline time.Time - if c.hc.Timeout > 0 { - deadline = sent.Add(c.hc.Timeout) + if timeout := c.httpClientFor(r).Timeout; timeout > 0 { + deadline = sent.Add(timeout) } reason := stallReason(err) if reason == "" || ctx.Err() != nil || (!deadline.IsZero() && !time.Now().Before(deadline)) { @@ -439,6 +476,30 @@ func (c *Client) roundTrip(ctx context.Context, r Request, u *url.URL, body []by return res, err } +// httpClientFor is the *http.Client one request is sent through: c.hc, or — +// when r.Timeout raises a bounded client's ceiling — a shallow copy of it with +// that longer Timeout. The copy shares c.hc's Transport (and so its connection +// pool and CA scope) and its forced CheckRedirect; only the whole-request +// deadline differs. +func (c *Client) httpClientFor(r Request) *http.Client { + if r.Timeout <= 0 || c.hc.Timeout <= 0 || r.Timeout <= c.hc.Timeout { + return c.hc + } + hc := *c.hc + hc.Timeout = r.Timeout + return &hc +} + +// largeApplyTimeout is the Request.Timeout for a batchApply request carrying n +// items: LargeApplyTimeout once n crosses the server's large-apply threshold, +// zero (the client's own ceiling) otherwise. +func largeApplyTimeout(n int) time.Duration { + if n > LargeApplyItemThreshold { + return LargeApplyTimeout + } + return 0 +} + // newRequest builds r's HTTP request and stamps it (see stampRequest). // roundTripOnce sends a copy of it per attempt. func (c *Client) newRequest(ctx context.Context, r Request, u *url.URL, body []byte) (*http.Request, error) { @@ -490,7 +551,7 @@ func (c *Client) roundTripOnce(ctx context.Context, r Request, u *url.URL, stamp } //nolint:gosec // G704: the URL is the workspace's configured server, resolved from the sidecar or --server-url; dialing it IS the feature. - resp, err := c.hc.Do(req) + resp, err := c.httpClientFor(r).Do(req) if ht.stop() { // The header timer fired first. Headers that raced it are discarded: // the attempt's context is already canceled, so its body is lost. diff --git a/internal/httpclient/wire/large_apply_timeout_test.go b/internal/httpclient/wire/large_apply_timeout_test.go new file mode 100644 index 000000000..6d9bdbcec --- /dev/null +++ b/internal/httpclient/wire/large_apply_timeout_test.go @@ -0,0 +1,135 @@ +// Contributed by gascity from bd-enterprise (internal/httpclient/wire/ent_large_apply_timeout_test.go@1ee63dff5) +// to OSS beads under the MIT license. +package wire + +import ( + "errors" + "net" + "net/http" + "testing" + "time" +) + +// A batchApply request over the server's large-apply threshold is dialed under +// LargeApplyTimeout, not the client's ordinary per-request ceiling. +// +// The fake server answers every write after slowWrite, which is longer than the +// injected client's own Timeout (shortCeiling) and far shorter than +// LargeApplyTimeout. So a request that stays under the client's ordinary +// ceiling fails with a Client.Timeout, and one the client raised to +// LargeApplyTimeout completes: the same shape as a real 1000-item apply on a +// loaded server, where bd serve grants five minutes and the client used to give +// up at DefaultTimeout (60s) while the server went on to commit. +const ( + shortCeiling = 200 * time.Millisecond + slowWrite = 1 * time.Second +) + +func slowWrites(contextJSON string) http.HandlerFunc { + fast := serveEverything(contextJSON) + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != PathContext { + select { + case <-time.After(slowWrite): + case <-r.Context().Done(): + return + } + } + fast(w, r) + } +} + +// newSlowWriteClient is a client whose own ceiling is shortCeiling, aimed at a +// server that answers every write after slowWrite. The handshake is primed +// under DefaultTimeout BEFORE the ceiling shrinks, so the only request +// shortCeiling ever bounds is the write under test: a context fetch stalled +// past it on a loaded host would otherwise fail an over-threshold case for the +// wrong reason, or hand an at-threshold control the *ConnectError it looks for. +func newSlowWriteClient(t *testing.T, caps ...string) (*Client, *recorder) { + t.Helper() + c, rec := newTestClient(t, Options{HTTPClient: &http.Client{Timeout: DefaultTimeout}}, nil, + slowWrites(contextBody("v0", "proj-1", caps...))) + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } + c.hc.Timeout = shortCeiling + return c, rec +} + +// wantClientTimeout requires err to be the client's own ceiling expiring on +// the op write itself: not the handshake, and not any other failure to connect. +func wantClientTimeout(t *testing.T, what, op string, err error) { + t.Helper() + var connErr *ConnectError + if !errors.As(err, &connErr) { + t.Fatalf("%s: err = %v (%T), want *ConnectError from the client's own ceiling", what, err, err) + } + if connErr.Op != op { + t.Fatalf("%s: the %s request failed (%v); want the %s write to reach the ceiling", what, connErr.Op, err, op) + } + var netErr net.Error + if !errors.As(err, &netErr) || !netErr.Timeout() { + t.Fatalf("%s: err = %v, want the client's own timeout", what, err) + } +} + +func TestApplyBatchOverTheLargeApplyThresholdOutlivesTheClientCeiling(t *testing.T) { + c, rec := newSlowWriteClient(t, withBatchApply(CapBatchApplyLarge)...) + n := LargeApplyItemThreshold + 1 + if _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(n)); err != nil { + t.Fatalf("ApplyBatch(%d items) against a server slower than the client's %v ceiling: %v; "+ + "a large apply must be dialed under LargeApplyTimeout", n, shortCeiling, err) + } + if got := rec.at(t, rec.count()-1).path; got != PathIssuesBatchApply { + t.Errorf("last request = %q, want %q", got, PathIssuesBatchApply) + } +} + +func TestApplyBatchAtTheLargeApplyThresholdKeepsTheClientCeiling(t *testing.T) { + // The control: at (not over) the threshold the server runs the request + // under its ORDINARY deadline, so the client keeps its ordinary ceiling + // and the same slow server times it out. + c, _ := newSlowWriteClient(t, withBatchApply(CapBatchApplyLarge)...) + _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(LargeApplyItemThreshold)) + wantClientTimeout(t, "ApplyBatch at the threshold", OpApplyBatch, err) +} + +func TestRequestTimeoutOnlyRaisesABoundedClient(t *testing.T) { + transport := &http.Transport{} + bounded := &http.Client{Timeout: DefaultTimeout, Transport: transport} + c, _ := newTestClient(t, Options{HTTPClient: bounded}, nil, serveEverything("{}")) + + if got := c.httpClientFor(Request{}); got != c.hc { + t.Error("a request with no Timeout must use the client's own *http.Client") + } + if got := c.httpClientFor(Request{Timeout: time.Second}); got != c.hc { + t.Error("a Timeout SHORTER than the client's must not lower it (that is the context's job)") + } + raised := c.httpClientFor(Request{Timeout: LargeApplyTimeout}) + if raised.Timeout != LargeApplyTimeout { + t.Errorf("raised Timeout = %v, want %v", raised.Timeout, LargeApplyTimeout) + } + if raised.Transport != transport { + t.Error("the raised client must share the client's Transport (its connection pool and CA scope)") + } + if raised.CheckRedirect == nil || !errors.Is(raised.CheckRedirect(nil, nil), http.ErrUseLastResponse) { + t.Error("the raised client must keep the forced CheckRedirect that refuses every redirect") + } + if c.hc.Timeout != DefaultTimeout { + t.Errorf("raising one request mutated the client's own Timeout to %v", c.hc.Timeout) + } + + unbounded, _ := newTestClient(t, Options{HTTPClient: &http.Client{}}, nil, serveEverything("{}")) + if got := unbounded.httpClientFor(Request{Timeout: LargeApplyTimeout}); got.Timeout != 0 { + t.Errorf("an unbounded client was given a %v ceiling; it must stay unbounded", got.Timeout) + } +} + +func TestLargeApplyTimeoutIsLongerThanTheDefault(t *testing.T) { + if LargeApplyTimeout <= DefaultTimeout { + t.Fatalf("LargeApplyTimeout = %v must exceed DefaultTimeout = %v", LargeApplyTimeout, DefaultTimeout) + } + if largeApplyTimeout(LargeApplyItemThreshold) != 0 || largeApplyTimeout(LargeApplyItemThreshold+1) != LargeApplyTimeout { + t.Fatal("largeApplyTimeout must switch exactly when the item count EXCEEDS LargeApplyItemThreshold") + } +} diff --git a/internal/httpclient/wire/stall_retry_test.go b/internal/httpclient/wire/stall_retry_test.go index 26f8d7079..32723e849 100644 --- a/internal/httpclient/wire/stall_retry_test.go +++ b/internal/httpclient/wire/stall_retry_test.go @@ -707,6 +707,59 @@ func TestAReadThatSpentTheWholeTimeoutIsNotRetried(t *testing.T) { } } +// TestARaisedRequestTimeoutAlsoBoundsTheStallRetry: the retry runs on what the +// first attempt left of the REQUEST's overall timeout, so a read that raises +// its ceiling (Request.Timeout) keeps the raised ceiling on its retry rather +// than falling back to the client's own. The server answers every request +// after slowRead, and the retry is sent once the header timer fires, so under +// the client's own ceiling (slowRead, counted from the first send) the retry +// runs out before its answer; under the raised one it gets it. +func TestARaisedRequestTimeoutAlsoBoundsTheStallRetry(t *testing.T) { + const slowRead = 2 * testHeaderTimeout + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-time.After(slowRead): + case <-r.Context().Done(): + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + base, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + var retries int + c, err := New(base, nil, Options{ + HTTPClient: &http.Client{Transport: srv.Client().Transport, Timeout: slowRead}, + ResponseHeaderTimeout: testHeaderTimeout, + OnStallRetry: func(StallRetry) { retries++ }, + }) + if err != nil { + t.Fatal(err) + } + read := Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext} + + // The control: the client's own ceiling ends before the retry's answer. + var hte *ResponseHeaderTimeoutError + if err := c.Do(ctx(t), read, nil); !errors.As(err, &hte) || !hte.Retried { + t.Fatalf("Do under the client's own %v ceiling: err = %v, want a retried *ResponseHeaderTimeoutError", slowRead, err) + } + + read.Timeout = DefaultTimeout + var out map[string]any + if err := c.Do(ctx(t), read, &out); err != nil { + t.Fatalf("Do with Request.Timeout %v: %v; the retry must run on the raised ceiling, not the client's %v", + read.Timeout, err, slowRead) + } + if out["ok"] != true { + t.Fatalf("body = %v, want the retry's answer", out) + } + if retries != 2 { + t.Errorf("OnStallRetry fired %d times, want 2 (one per Do)", retries) + } +} + // countingToken is a staticToken that counts its Authorize calls. type countingToken struct { staticToken diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index 49417a79a..b28f136d2 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -539,7 +539,11 @@ func (c *Client) ApplyBatch(ctx context.Context, body ApplyBatchRequest) (*apige } var out apigen.ApplyBatchResponse - r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body, Actor: body.Actor} + // A plan over the server's large-apply threshold runs under the server's + // extended budget, so it is dialed under the matching client ceiling + // rather than DefaultTimeout. + r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body, Actor: body.Actor, + Timeout: largeApplyTimeout(len(body.Items))} if err := c.dispatch(ctx, r, &out); err != nil { return nil, err }