From d7f974d6f8480ed3bdd8e54490d672f8c5fe8615 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Wed, 30 Sep 2026 20:19:08 +0000 Subject: [PATCH 1/5] test(storage): measure ApplyBatchInTx SQL statement counts (B0) Adds a driver.Connector-level counting/latency-injecting wrapper (internal/storage/sqlcount) plus shared plan-request fixtures (internal/storage/batchfixtures: mol-normalized 356/712-item shapes and a 40-item classic shape) and backend-specific pinning tests + benchmarks (internal/storage/embeddeddolt, internal/storage/dolt) that measure the ACTUAL number of SQL statements issueops.ApplyBatchInTx issues for each shape, on both the embedded and Dolt (TCP) backends. Measured totals (Prepare+Exec+Query+StmtExec+StmtQuery inside one transaction): 356 items ~7008-7009, 712 items 14014-14015, 40 items 846, identical on both backends (expected: statement issuance is driven by Go application logic, not backend-specific SQL planning). Pinned with a small (+/-2) tolerance to absorb an observed, host-contention-driven +/-1 Query drift (Exec is deterministic on every run). This is the regression baseline slice B2 (a later, lighter fast path) must demonstrably lower. batchfixtures.UncappedPlan bypasses issueops.MaxApplyBatchItems so this measurement does not depend on slice B1's cap raise. Co-Authored-By: Claude Sonnet 5 --- .../storage/batchfixtures/batchfixtures.go | 241 ++++++++++ .../batchfixtures/batchfixtures_test.go | 110 +++++ .../storage/batchfixtures/uncapped_plan.go | 45 ++ .../dolt/large_batch_apply_measure_test.go | 201 +++++++++ .../large_batch_apply_measure_test.go | 300 +++++++++++++ internal/storage/sqlcount/sqlcount.go | 418 ++++++++++++++++++ 6 files changed, 1315 insertions(+) create mode 100644 internal/storage/batchfixtures/batchfixtures.go create mode 100644 internal/storage/batchfixtures/batchfixtures_test.go create mode 100644 internal/storage/batchfixtures/uncapped_plan.go create mode 100644 internal/storage/dolt/large_batch_apply_measure_test.go create mode 100644 internal/storage/embeddeddolt/large_batch_apply_measure_test.go create mode 100644 internal/storage/sqlcount/sqlcount.go diff --git a/internal/storage/batchfixtures/batchfixtures.go b/internal/storage/batchfixtures/batchfixtures.go new file mode 100644 index 000000000..fb286e987 --- /dev/null +++ b/internal/storage/batchfixtures/batchfixtures.go @@ -0,0 +1,241 @@ +// Package batchfixtures builds issueops.ApplyBatchRequest values shaped like +// the plans S5b-3-LARGE-BATCH-DESIGN.md measures: the mol-normalized +// issue/PR work plan (and its 2x variant) and a "classic" hand-authored +// plan. It does no I/O — every fixture here is pure data construction — so a +// benchmark or test in any backend package can import it without risking an +// import cycle with internal/storage/issueops (which the backend packages +// already import for ApplyBatchInTx itself). +package batchfixtures + +import ( + "encoding/json" + "fmt" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// createKey names the Nth created work item across every shape here, so a +// caller inspecting a failure can tell which fixture minted a given key. +func createKey(prefix string, n int) string { + return fmt.Sprintf("%s-w%d", prefix, n) +} + +// molCounts is the sub-item composition mol-normalized-issue-pr-work-v1 +// measures at 1x: 102 creates, 136 blocks edges, 102 tracks-to-root edges, +// 16 assign updates == 356 items total. See S5b-3-LARGE-BATCH-DESIGN.md +// "Measured sizes". +type molCounts struct { + creates int + blocksEdges int + tracksEdges int + assignItems int +} + +var mol1x = molCounts{creates: 102, blocksEdges: 136, tracksEdges: 102, assignItems: 16} + +// mol2x is the design's explicit 2x test point (712 items): every +// sub-category of mol1x doubled, rather than a differently-shaped plan. That +// is a modeling decision the design doc leaves open ("...2x the above +// shape") — doubling each sub-count is the literal reading and keeps the +// edge/create ratio identical between the two measured shapes, which is what +// makes them comparable points on the same curve. +var mol2x = molCounts{creates: 204, blocksEdges: 272, tracksEdges: 204, assignItems: 32} + +// buildMolShape builds a mol-normalized-issue-pr-work-v1-shaped request: a +// flat batch of work-item creates (each carrying metadata), a blocks chain +// wiring them together, every created item tracking back to an +// already-existing root issue, and a tail of assignee updates on a subset of +// the created items. +// +// rootID must already exist in the target store — it is NOT created by this +// request, matching the shape's own description ("...tracks to the root"): +// the root is the pre-existing issue or PR the batch's new work tracks +// against, not itself a member of the batch. +func buildMolShape(actor, prefix string, c molCounts, rootID string) issueops.ApplyBatchRequest { + items := make([]issueops.ApplyItem, 0, c.creates+c.blocksEdges+c.tracksEdges+c.assignItems) + + // Creates, each carrying a small metadata blob — "creates with metadata" + // per the design's shape description. + for i := 1; i <= c.creates; i++ { + key := createKey(prefix, i) + meta, err := json.Marshal(map[string]any{ + "source": "mol-normalized-issue-pr-work-v1", + "seq": i, + }) + if err != nil { + panic(fmt.Sprintf("batchfixtures: marshal create metadata: %v", err)) + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{ + Key: key, + Issue: &types.Issue{ + Title: fmt.Sprintf("%s work item %d", prefix, i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: meta, + }, + }, + }) + } + + // Blocks edges: a deterministic, duplicate-free set of (source, source+skip) + // pairs over the created keys, generated by widening the skip distance + // once every (creates-1) edges are exhausted at the current skip. Every + // pair names two ALREADY-DECLARED create keys (both appear earlier in + // Items), which is what a backward-only Ref requires. + edgesAdded := 0 + for skip := 1; edgesAdded < c.blocksEdges; skip++ { + for i := 1; i+skip <= c.creates && edgesAdded < c.blocksEdges; i++ { + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, i)}, + Target: issueops.Ref{Key: createKey(prefix, i+skip)}, + Type: issueops.DepBlocks, + }, + }) + edgesAdded++ + } + } + + // Tracks-to-root edges: every created item tracks the pre-existing root. + for i := 1; i <= c.tracksEdges; i++ { + srcN := ((i - 1) % c.creates) + 1 + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, srcN)}, + Target: issueops.Ref{ID: rootID}, + Type: issueops.DepTracks, + }, + }) + } + + // Assign updates on a subset of the created items. + for i := 1; i <= c.assignItems; i++ { + assignee := fmt.Sprintf("agent-%d", i) + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: createKey(prefix, i)}, + Patch: issueops.IssuePatch{ + Assignee: issueops.Field[string]{Set: true, Value: assignee}, + }, + }, + }) + } + + return issueops.ApplyBatchRequest{ + Actor: actor, + Items: items, + Provenance: fmt.Sprintf("batchfixtures: mol-normalized-issue-pr-work-v1 (%d items)", len(items)), + } +} + +// Shape356 builds the design's primary measured shape: 102 creates with +// metadata, 238 edges (136 blocks + 102 tracks-to-root) and 16 assign +// updates, for 356 items total. rootID must already exist in the target +// store. +func Shape356(actor, rootID string) issueops.ApplyBatchRequest { + return buildMolShape(actor, "mol1x", mol1x, rootID) +} + +// Shape712 builds the design's 2x test point: every Shape356 sub-category +// doubled (204 creates, 476 edges [272 blocks + 204 tracks], 32 assign +// updates), for 712 items total. rootID must already exist in the target +// store. +func Shape712(actor, rootID string) issueops.ApplyBatchRequest { + return buildMolShape(actor, "mol2x", mol2x, rootID) +} + +// ShapeClassic40 builds a 40-item "classic" (non-mol) plan: 10 creates, 20 +// edges (12 blocks + 8 parent-child) and 10 assign updates. +// +// The design doc does not pin exact sub-counts for the classic shape — it +// only describes MC's classic formulas qualitatively as "at most 9 source +// steps, so roughly 40 items or fewer". This composition is this +// implementation's modeling decision for a comparison point below the mol +// shape's edge-heavy ratio: parent-child plus blocks rather than +// tracks-to-root, because a classic (non-convoy) plan has no single root +// every item reports to. +func ShapeClassic40(actor string) issueops.ApplyBatchRequest { + const ( + creates = 10 + blocks = 12 + parentKids = 8 + assigns = 10 + ) + prefix := "classic" + items := make([]issueops.ApplyItem, 0, creates+blocks+parentKids+assigns) + + for i := 1; i <= creates; i++ { + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{ + Key: createKey(prefix, i), + Issue: &types.Issue{ + Title: fmt.Sprintf("%s item %d", prefix, i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }, + }, + }) + } + + edgesAdded := 0 + for skip := 1; edgesAdded < blocks; skip++ { + for i := 1; i+skip <= creates && edgesAdded < blocks; i++ { + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, i)}, + Target: issueops.Ref{Key: createKey(prefix, i+skip)}, + Type: issueops.DepBlocks, + }, + }) + edgesAdded++ + } + } + + // Parent-child edges: item 1 is the parent of item 4 through creates, + // cycling within that range if parentKids exceeds it. Items 2 and 3 are + // skipped deliberately — the blocks loop above always wires (1,2) and + // (1,3) first (skip=1's first pair and skip=2's first pair), and a pair + // may carry only one dependency TYPE: a parent-child edge on the same + // pair a blocks edge already claimed is refused as a type conflict, not + // silently added as a second edge. + for i := 1; i <= parentKids; i++ { + childN := 4 + ((i - 1) % (creates - 3)) // 4..creates, cycling + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, 1)}, + Target: issueops.Ref{Key: createKey(prefix, childN)}, + Type: issueops.DepParentChild, + }, + }) + } + + for i := 1; i <= assigns; i++ { + assignee := fmt.Sprintf("agent-%d", i) + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: createKey(prefix, i)}, + Patch: issueops.IssuePatch{ + Assignee: issueops.Field[string]{Set: true, Value: assignee}, + }, + }, + }) + } + + return issueops.ApplyBatchRequest{ + Actor: actor, + Items: items, + Provenance: fmt.Sprintf("batchfixtures: classic (%d items)", len(items)), + } +} diff --git a/internal/storage/batchfixtures/batchfixtures_test.go b/internal/storage/batchfixtures/batchfixtures_test.go new file mode 100644 index 000000000..fa184b5ef --- /dev/null +++ b/internal/storage/batchfixtures/batchfixtures_test.go @@ -0,0 +1,110 @@ +package batchfixtures_test + +import ( + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/issueops" +) + +// TestShapeItemCounts pins the exact item count and per-kind composition of +// each fixture shape, so a change to batchfixtures itself (not the storage +// layer under test) is caught here rather than surfacing as a confusing +// statement-count drift in the backend-level pinning tests. +func TestShapeItemCounts(t *testing.T) { + const rootID = "root-1" + + tests := []struct { + name string + req issueops.ApplyBatchRequest + wantTotal int + wantCreate int + wantDepAdd int + wantUpdate int + wantClose int + }{ + { + name: "356 (mol 1x)", + req: batchfixtures.Shape356("tester", rootID), + wantTotal: 356, + wantCreate: 102, + wantDepAdd: 238, + wantUpdate: 16, + }, + { + name: "712 (mol 2x)", + req: batchfixtures.Shape712("tester", rootID), + wantTotal: 712, + wantCreate: 204, + wantDepAdd: 476, + wantUpdate: 32, + }, + { + name: "40 (classic)", + req: batchfixtures.ShapeClassic40("tester"), + wantTotal: 40, + wantCreate: 10, + wantDepAdd: 20, + wantUpdate: 10, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := len(tc.req.Items); got != tc.wantTotal { + t.Fatalf("len(Items) = %d, want %d", got, tc.wantTotal) + } + var creates, depAdds, updates, closes int + for _, item := range tc.req.Items { + switch item.Kind { + case issueops.ItemCreate: + creates++ + case issueops.ItemDepAdd: + depAdds++ + case issueops.ItemUpdate: + updates++ + case issueops.ItemClose: + closes++ + } + } + if creates != tc.wantCreate { + t.Errorf("creates = %d, want %d", creates, tc.wantCreate) + } + if depAdds != tc.wantDepAdd { + t.Errorf("dep_adds = %d, want %d", depAdds, tc.wantDepAdd) + } + if updates != tc.wantUpdate { + t.Errorf("updates = %d, want %d", updates, tc.wantUpdate) + } + if closes != tc.wantClose { + t.Errorf("closes = %d, want %d", closes, tc.wantClose) + } + + // Every shape must validate and plan cleanly against the role's own + // request-shape contract (ref ordering, key uniqueness, + // exactly-one-payload) whenever it fits under the role's item cap — + // a fixture that failed it would be measuring nothing. The 356- and + // 712-item shapes exceed issueops.MaxApplyBatchItems (100) before B1 + // raises it to 1000, so this check is skipped above the current cap + // rather than ordered against a B1 commit that may not have landed + // yet; B1's own tests cover the cap itself. + if len(tc.req.Items) > issueops.MaxApplyBatchItems { + t.Skipf("item count %d exceeds current issueops.MaxApplyBatchItems (%d); cap coverage lives in the B1 tests", + len(tc.req.Items), issueops.MaxApplyBatchItems) + } + if _, err := storage.PlanApplyBatch(tc.req); err != nil { + t.Fatalf("PlanApplyBatch: %v", err) + } + }) + } +} + +// TestShapeClassic40NoRootRequired documents that the classic shape, unlike +// the mol shapes, needs no pre-existing root issue. +func TestShapeClassic40NoRootRequired(t *testing.T) { + req := batchfixtures.ShapeClassic40("tester") + if _, err := storage.PlanApplyBatch(req); err != nil { + t.Fatalf("PlanApplyBatch: %v", err) + } +} diff --git a/internal/storage/batchfixtures/uncapped_plan.go b/internal/storage/batchfixtures/uncapped_plan.go new file mode 100644 index 000000000..b28e25083 --- /dev/null +++ b/internal/storage/batchfixtures/uncapped_plan.go @@ -0,0 +1,45 @@ +package batchfixtures + +import ( + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// UncappedPlan builds a storage.ApplyBatchPlan directly from req, bypassing +// storage.PlanApplyBatch's issueops.MaxApplyBatchItems cap check. +// +// IT EXISTS SO B0's STATEMENT-COUNT MEASUREMENT CAN RUN BEFORE B1 RAISES THE +// CAP. issueops.ApplyBatchInTx itself enforces no item-count limit — the +// limit is a request-validation POLICY that PlanApplyBatch applies before a +// plan ever reaches ApplyBatchInTx — so measuring ApplyBatchInTx's real +// statement count on a 356- or 712-item plan is a true measurement of the +// mechanism either way. Routing it through this helper rather than +// PlanApplyBatch means the B0 measurement commit does not have to be +// sequenced after, or squashed with, the B1 commit that raises the cap. +// +// It replicates PlanApplyBatch's key-index construction — the one piece of +// its output ApplyBatchInTx's own key resolution depends on — but NONE of +// its other validation (ref ordering, guard rules, edge-metadata +// normalization). A caller building a request for this helper must already +// satisfy every other rule PlanApplyBatch checks; every shape in this +// package does, which TestShapeItemCounts confirms for the one shape +// (ShapeClassic40) small enough to run through PlanApplyBatch directly under +// the current, pre-B1 cap. +func UncappedPlan(req issueops.ApplyBatchRequest) storage.ApplyBatchPlan { + keyIndex := make(map[string]int, len(req.Items)) + for i, item := range req.Items { + if item.Kind == issueops.ItemCreate && item.Create != nil && item.Create.Key != "" { + keyIndex[item.Create.Key] = i + } + } + items := make([]issueops.ApplyItem, len(req.Items)) + copy(items, req.Items) + return storage.ApplyBatchPlan{ + Actor: req.Actor, + Provenance: req.Provenance, + ForceIDPrefix: req.ForceIDPrefix, + SkipPerEdgeCycleCheck: req.SkipPerEdgeCycleCheck, + Items: items, + KeyIndex: keyIndex, + } +} diff --git a/internal/storage/dolt/large_batch_apply_measure_test.go b/internal/storage/dolt/large_batch_apply_measure_test.go new file mode 100644 index 000000000..db60b6a9f --- /dev/null +++ b/internal/storage/dolt/large_batch_apply_measure_test.go @@ -0,0 +1,201 @@ +package dolt + +// B0 (S5b-3-LARGE-BATCH-DESIGN.md): measure the ACTUAL number of SQL +// statements issueops.ApplyBatchInTx issues on the Dolt server (TCP) +// backend, for the design's three measured plan shapes. See +// internal/storage/embeddeddolt/large_batch_apply_measure_test.go for the +// embedded backend equivalent, and internal/storage/batchfixtures for the +// shared, backend-agnostic plan construction. +// +// This file opens a SECOND, counting-wrapped connection using the same +// mysql.Config a normal *DoltStore already parsed from its own connStr +// (store.connStr, set by newServerMode/openServerConnection) — because +// issueops.ApplyBatchInTx takes a concrete *sql.Tx and the store's own pool +// is not wrapped. It is package dolt (not dolt_test) specifically to reach +// that unexported field, the same way dolt_test.go's own helpers already do. + +import ( + "context" + "database/sql" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// openCountedDoltConn opens a second, counting-wrapped connection to the +// same Dolt server database an already-open *DoltStore is using, by +// re-parsing that store's own connStr. MaxOpenConns is pinned to 1 so every +// statement observed goes over one physical connection — matching how +// ApplyBatchInTx itself runs entirely inside one *sql.Tx on one connection. +func openCountedDoltConn(connStr string, counts *sqlcount.Counts) (*sql.DB, func(), error) { + cfg, err := mysql.ParseDSN(connStr) + if err != nil { + return nil, nil, err + } + connector, err := mysql.NewConnector(cfg) + if err != nil { + return nil, nil, err + } + db := sql.OpenDB(sqlcount.WrapConnector(connector, counts)) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + return db, func() { db.Close() }, nil +} + +// measureDoltApply builds plan via batchfixtures.UncappedPlan (bypassing +// issueops.MaxApplyBatchItems, see uncapped_plan.go), runs it through +// issueops.ApplyBatchInTx inside one transaction on a fresh counted +// connection, and returns the statement-count snapshot. counts.Reset() is +// called immediately before BeginTx so the snapshot reflects only the +// transaction itself, not connection setup. +func measureDoltApply(ctx context.Context, connStr, rootID string, counts *sqlcount.Counts, build func(rootID string) issueops.ApplyBatchRequest) (sqlcount.Snapshot, error) { + db, closeDB, err := openCountedDoltConn(connStr, counts) + if err != nil { + return sqlcount.Snapshot{}, err + } + defer closeDB() + + plan := batchfixtures.UncappedPlan(build(rootID)) + + counts.Reset() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return sqlcount.Snapshot{}, err + } + if _, _, err := storageissueops.ApplyBatchInTx(ctx, tx, plan); err != nil { + _ = tx.Rollback() + return sqlcount.Snapshot{}, err + } + if err := tx.Commit(); err != nil { + return sqlcount.Snapshot{}, err + } + return counts.Snapshot(), nil +} + +// largeBatchApplyDoltShapes mirrors embeddeddolt's largeBatchApplyShapes: +// the three shapes the design measures, plus a per-shape database name (this +// backend uses one fresh throwaway database per shape via +// setupConcurrentTestStore, not a directory clone). +var largeBatchApplyDoltShapes = []struct { + name string + build func(rootID string) issueops.ApplyBatchRequest +}{ + { + name: "356 (mol 1x)", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape356("tester", rootID) + }, + }, + { + name: "712 (mol 2x)", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape712("tester", rootID) + }, + }, + { + name: "40 (classic)", + build: func(string) issueops.ApplyBatchRequest { + return batchfixtures.ShapeClassic40("tester") + }, + }, +} + +// TestLargeBatchApplyStatementCounts_Dolt pins the ACTUAL number of SQL +// statements issueops.ApplyBatchInTx issues on the Dolt server (TCP) +// backend, for each of the design's three measured shapes +// (S5b-3-LARGE-BATCH-DESIGN.md, slice B0). Sibling regression baseline to +// TestLargeBatchApplyStatementCounts_Embedded in the embeddeddolt package — +// see that test's doc comment for why counts are pinned with a small +// tolerance rather than exactly, and why a drift beyond it should be +// re-measured and re-pinned deliberately rather than loosened further. +func TestLargeBatchApplyStatementCounts_Dolt(t *testing.T) { + ctx := context.Background() + + for _, tc := range largeBatchApplyDoltShapes { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, root, "tester"); err != nil { + t.Fatalf("create root issue: %v", err) + } + + var counts sqlcount.Counts + got, err := measureDoltApply(ctx, store.connStr, root.ID, &counts, tc.build) + if err != nil { + t.Fatalf("measureDoltApply: %v", err) + } + t.Logf("dolt %s: Prepare=%d Exec=%d Query=%d StmtExec=%d StmtQuery=%d Total=%d", + tc.name, got.Prepare, got.Exec, got.Query, got.StmtExec, got.StmtQuery, got.Total()) + + want, ok := pinnedDoltStatementCounts[tc.name] + if !ok { + t.Fatalf("no pinned count for shape %q; run with -v, read the logged Total, and add it to pinnedDoltStatementCounts", tc.name) + } + diff := got.Total() - want + if diff < 0 { + diff = -diff + } + if diff > statementCountToleranceDolt { + t.Errorf("dolt %s: Total() = %d, want pinned %d +/- %d (see large_batch_apply_measure_test.go)", + tc.name, got.Total(), want, statementCountToleranceDolt) + } + }) + } +} + +// statementCountToleranceDolt mirrors embeddeddolt's statementCountTolerance: +// a narrow allowance for occasional host-contention-driven retry polling, +// not a license to loosen a real drift. See that constant's doc comment. +const statementCountToleranceDolt = 2 + +// pinnedDoltStatementCounts is the B0 regression baseline for the Dolt +// server (TCP) backend: the Total() statement count issueops.ApplyBatchInTx +// issued for each measured shape, as last observed on this branch. B2 must +// lower these; a change for any other reason should be re-measured and +// re-pinned deliberately, not adjusted to make a failure go away. +var pinnedDoltStatementCounts = map[string]int64{ + "356 (mol 1x)": 7008, + "712 (mol 2x)": 14014, + "40 (classic)": 846, +} + +// BenchmarkLargeBatchApply_Dolt is gated by setupBenchStore's own +// BEADS_BENCH_DOLT_PORT opt-in (benchDoltServerPort) — never resolved from +// ambient BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT, which a gc agent shell may +// point at a shared production server (be-cfm3z). +func BenchmarkLargeBatchApply_Dolt(b *testing.B) { + for _, tc := range largeBatchApplyDoltShapes { + b.Run(tc.name, func(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + ctx := context.Background() + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, root, "bench"); err != nil { + b.Fatalf("create root issue: %v", err) + } + + var counts sqlcount.Counts + var lastTotal int64 + b.ResetTimer() + for i := 0; i < b.N; i++ { + got, err := measureDoltApply(ctx, store.connStr, root.ID, &counts, tc.build) + if err != nil { + b.Fatalf("measureDoltApply: %v", err) + } + lastTotal = got.Total() + } + b.StopTimer() + b.ReportMetric(float64(lastTotal), "statements/op") + }) + } +} diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go new file mode 100644 index 000000000..200174b58 --- /dev/null +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -0,0 +1,300 @@ +//go:build cgo + +package embeddeddolt_test + +// B0 (S5b-3-LARGE-BATCH-DESIGN.md): measure the ACTUAL number of SQL +// statements issueops.ApplyBatchInTx issues on the embedded Dolt backend, +// for the design's three measured plan shapes. See +// internal/storage/dolt/large_batch_apply_measure_test.go for the Dolt +// server (TCP) backend equivalent, and internal/storage/batchfixtures for +// the shared, backend-agnostic plan construction. +// +// This file opens a SECOND, counting-wrapped connection against an on-disk +// embedded directory a normal *EmbeddedDoltStore already migrated — exactly +// the pattern testEnv.exec/queryScalar in create_issue_test.go use for raw +// verification queries — because issueops.ApplyBatchInTx takes a concrete +// *sql.Tx and the store's own connections are not wrapped. It duplicates +// open.go's buildDSN rather than modifying OpenSQL to accept a connector +// hook: that would add a test-only seam to a production construction path +// for the sake of one measurement harness. + +import ( + "context" + "database/sql" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + doltembed "github.com/dolthub/driver/v2" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// measureCommitName/measureCommitEmail mirror open.go's commitName/commitEmail, +// duplicated because those are unexported. +const ( + measureCommitName = "beads" + measureCommitEmail = "beads@local" +) + +// measureDSN replicates open.go's buildDSN for a raw, counting-wrapped +// connection to an already-migrated embedded directory. +func measureDSN(dir, database string) string { + v := url.Values{} + v.Set(doltembed.CommitNameParam, measureCommitName) + v.Set(doltembed.CommitEmailParam, measureCommitEmail) + v.Set(doltembed.MultiStatementsParam, "true") + if strings.TrimSpace(database) != "" { + v.Set(doltembed.DatabaseParam, database) + } + path := dir + if os.PathSeparator == '\\' { + path = strings.ReplaceAll(path, `\`, `/`) + } + return "file://" + path + "?" + v.Encode() +} + +// openCountedConn opens a counting-wrapped *sql.DB against dataDir and USEs +// database on it. The returned cleanup closes both the db and the +// connector; the caller drives transactions on the returned db itself. +func openCountedConn(ctx context.Context, dataDir, database string, counts *sqlcount.Counts) (*sql.DB, func(), error) { + cfg, err := doltembed.ParseDSN(measureDSN(dataDir, database)) + if err != nil { + return nil, nil, err + } + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 0 + bo.MaxInterval = 5 * time.Second + cfg.BackOff = bo + + connector, err := doltembed.NewConnector(cfg) + if err != nil { + return nil, nil, err + } + wrapped := sqlcount.WrapConnector(connector, counts) + db := sql.OpenDB(wrapped) + // A single connection: PrepareContext/BeginTx on this db must land on + // the same embedded session for USE below to still apply when the + // measured transaction begins. + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + cleanup := func() { + _ = db.Close() + _ = connector.Close() + } + + if err := db.PingContext(ctx); err != nil { + cleanup() + return nil, nil, err + } + if _, err := db.ExecContext(ctx, "USE `"+database+"`"); err != nil { + cleanup() + return nil, nil, err + } + return db, cleanup, nil +} + +// measureApply runs build(rootID) through issueops.ApplyBatchInTx on a +// counting-wrapped connection to dataDir/database — which must already be a +// migrated embedded Dolt database containing an issue at rootID, or "" if +// the shape needs no root (ShapeClassic40) — and returns the resulting +// statement counts. Everything before the returned counts.Reset() point is +// setup and excluded; only the transaction ApplyBatchInTx runs inside is +// measured. +func measureApply(ctx context.Context, dataDir, database, rootID string, build func(rootID string) issueops.ApplyBatchRequest) (sqlcount.Snapshot, error) { + counts := &sqlcount.Counts{} + db, cleanup, err := openCountedConn(ctx, dataDir, database, counts) + if err != nil { + return sqlcount.Snapshot{}, err + } + defer cleanup() + + plan := batchfixtures.UncappedPlan(build(rootID)) + counts.Reset() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return sqlcount.Snapshot{}, err + } + if _, _, err := storageissueops.ApplyBatchInTx(ctx, tx, plan); err != nil { + _ = tx.Rollback() + return sqlcount.Snapshot{}, err + } + if err := tx.Commit(); err != nil { + return sqlcount.Snapshot{}, err + } + return counts.Snapshot(), nil +} + +// largeBatchApplyShapes is the design's three measured shapes, shared by the +// pinning test and the benchmark below. +var largeBatchApplyShapes = []struct { + name string + db string + build func(rootID string) issueops.ApplyBatchRequest +}{ + { + name: "356 (mol 1x)", + db: "m356", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape356("tester", rootID) + }, + }, + { + name: "712 (mol 2x)", + db: "m712", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape712("tester", rootID) + }, + }, + { + name: "40 (classic)", + db: "mclassic", + build: func(string) issueops.ApplyBatchRequest { + return batchfixtures.ShapeClassic40("tester") + }, + }, +} + +// TestLargeBatchApplyStatementCounts_Embedded pins the ACTUAL number of SQL +// statements issueops.ApplyBatchInTx issues on the embedded backend, for +// each of the design's three measured shapes (S5b-3-LARGE-BATCH-DESIGN.md, +// slice B0). It is a REGRESSION BASELINE for B2 (a later, lighter fast +// path): B2 must lower these numbers, and this test is what proves it did. +// +// The exact counts are backend- and Dolt-version-sensitive by nature — they +// come from real driver round trips, not a cost model — so a failure here +// after an unrelated Dolt/driver upgrade is expected and should be +// re-pinned with a note in the commit, not silently loosened to a range. +// +// statementCountTolerance absorbs one narrow, observed source of +// non-determinism: under host contention, the embedded engine occasionally +// issues one extra (or one fewer) internal Query round trip on the largest +// shape — most likely a lock-wait/retry poll, since the exact same input +// plan otherwise executes identically. Exec was observed PERFECTLY +// deterministic across >10 repeated runs of every shape; only Query ever +// moved, and only by 1, and only occasionally (roughly 1 run in 9 on the +// 356-item shape; 712 and classic-40 were never observed to vary). A wider +// drift than this tolerance is a real regression/improvement, not jitter, +// and must be re-measured and re-pinned deliberately. +const statementCountTolerance = 2 + +func TestLargeBatchApplyStatementCounts_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + + for _, tc := range largeBatchApplyShapes { + t.Run(tc.name, func(t *testing.T) { + fixture := newPristineEmbeddedDoltFixture(t, tc.db) + t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := fixture.store.CreateIssue(ctx, root, "tester"); err != nil { + t.Fatalf("create root issue: %v", err) + } + + got, err := measureApply(ctx, fixture.dataDir, fixture.database, root.ID, tc.build) + if err != nil { + t.Fatalf("measureApply: %v", err) + } + t.Logf("embedded %s: Prepare=%d Exec=%d Query=%d StmtExec=%d StmtQuery=%d Total=%d", + tc.name, got.Prepare, got.Exec, got.Query, got.StmtExec, got.StmtQuery, got.Total()) + + want, ok := pinnedEmbeddedStatementCounts[tc.name] + if !ok { + t.Fatalf("no pinned count for shape %q; run with -v, read the logged Total, and add it to pinnedEmbeddedStatementCounts", tc.name) + } + diff := got.Total() - want + if diff < 0 { + diff = -diff + } + if diff > statementCountTolerance { + t.Errorf("embedded %s: Total() = %d, want pinned %d +/- %d (see large_batch_apply_measure_test.go)", + tc.name, got.Total(), want, statementCountTolerance) + } + }) + } +} + +// pinnedEmbeddedStatementCounts is the B0 regression baseline for the +// embedded backend: the Total() statement count issueops.ApplyBatchInTx +// issued for each measured shape, as last observed on this branch. B2 must +// lower these; a change for any other reason should be re-measured and +// re-pinned deliberately, not adjusted to make a failure go away. +var pinnedEmbeddedStatementCounts = map[string]int64{ + "356 (mol 1x)": 7009, + "712 (mol 2x)": 14014, + "40 (classic)": 846, +} + +// BenchmarkLargeBatchApply_Embedded benchmarks issueops.ApplyBatchInTx on +// the embedded backend for each measured shape and reports the statement +// count alongside the usual timing, via b.ReportMetric. Each iteration +// builds and migrates a fresh, throwaway embedded database — not the +// pristine-template-clone fixture the *_test.go suite uses, because that +// helper is *testing.T-only — so per-iteration time includes that setup; +// -benchtime=Nx therefore matters more than a wall-clock target here. +func BenchmarkLargeBatchApply_Embedded(b *testing.B) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + b.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt benchmarks") + } + ctx := context.Background() + + for _, shape := range largeBatchApplyShapes { + b.Run(shape.name, func(b *testing.B) { + var totalStatements int64 + for i := 0; i < b.N; i++ { + b.StopTimer() + beadsDir, err := os.MkdirTemp("", "beads-bench-embedded-*") + if err != nil { + b.Fatalf("MkdirTemp: %v", err) + } + store, err := embeddeddolt.Open(ctx, beadsDir, shape.db, "main") + if err != nil { + os.RemoveAll(beadsDir) + b.Fatalf("Open: %v", err) + } + if err := store.SetConfig(ctx, "issue_prefix", shape.db); err != nil { + store.Close() + os.RemoveAll(beadsDir) + b.Fatalf("SetConfig: %v", err) + } + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, root, "tester"); err != nil { + store.Close() + os.RemoveAll(beadsDir) + b.Fatalf("create root issue: %v", err) + } + dataDir := filepath.Join(beadsDir, "embeddeddolt") + b.StartTimer() + + got, err := measureApply(ctx, dataDir, shape.db, root.ID, shape.build) + + b.StopTimer() + if err != nil { + store.Close() + os.RemoveAll(beadsDir) + b.Fatalf("measureApply: %v", err) + } + totalStatements += got.Total() + store.Close() + os.RemoveAll(beadsDir) + b.StartTimer() + } + if b.N > 0 { + b.ReportMetric(float64(totalStatements)/float64(b.N), "statements/op") + } + }) + } +} diff --git a/internal/storage/sqlcount/sqlcount.go b/internal/storage/sqlcount/sqlcount.go new file mode 100644 index 000000000..75d5f1516 --- /dev/null +++ b/internal/storage/sqlcount/sqlcount.go @@ -0,0 +1,418 @@ +// Package sqlcount wraps a database/sql/driver.Connector so a test or +// benchmark can count how many SQL statements a code path actually issues +// against a real driver, and optionally inject artificial per-statement +// latency to approximate a database that is not co-located with the caller. +// +// It exists for the large-batch-apply measurement work +// (S5b-3-LARGE-BATCH-DESIGN.md, slice B0): issueops.ApplyBatchInTx takes a +// concrete *sql.Tx, not an interface, so the only seam available to count its +// real round trips is the database/sql/driver boundary itself. Wrapping the +// driver.Connector a backend already builds — mysql.NewConnector for the Dolt +// server backend, doltembed.NewConnector for the embedded backend — is the +// one construction point both backends share, so one wrapper measures both +// without touching either driver or any production code path. +package sqlcount + +import ( + "context" + "database/sql/driver" + "fmt" + "sync/atomic" + "time" +) + +// Counts tracks how many times each driver entry point that represents one +// SQL statement or round trip fired. The zero value is ready to use. Safe +// for concurrent use. +type Counts struct { + prepare int64 + exec int64 + query int64 + stmtExec int64 + stmtQuery int64 + begin int64 + commit int64 + rollback int64 +} + +// Snapshot is a point-in-time, plain copy of Counts, safe to diff, log or +// assert on without synchronization. +type Snapshot struct { + Prepare int64 + Exec int64 + Query int64 + StmtExec int64 + StmtQuery int64 + Begin int64 + Commit int64 + Rollback int64 +} + +// Total is every round trip this wrapper attributes to an executed SQL +// statement: a Prepare, a driver-level Exec/Query issued without a separate +// prepare, or an Exec/Query against an already-prepared statement. Begin, +// Commit and Rollback are reported on Snapshot but excluded here, because a +// caller measuring one transaction's BODY (issueops.ApplyBatchInTx, for +// instance) opens and closes that transaction outside the measured window. +func (s Snapshot) Total() int64 { + return s.Prepare + s.Exec + s.Query + s.StmtExec + s.StmtQuery +} + +// Sub returns the per-field difference s-prior, for isolating one call's +// contribution on a longer-lived connection. +func (s Snapshot) Sub(prior Snapshot) Snapshot { + return Snapshot{ + Prepare: s.Prepare - prior.Prepare, + Exec: s.Exec - prior.Exec, + Query: s.Query - prior.Query, + StmtExec: s.StmtExec - prior.StmtExec, + StmtQuery: s.StmtQuery - prior.StmtQuery, + Begin: s.Begin - prior.Begin, + Commit: s.Commit - prior.Commit, + Rollback: s.Rollback - prior.Rollback, + } +} + +// Snapshot reads every counter. The read is not atomic ACROSS fields — a +// concurrent writer could interleave between two loads — which matches every +// other multi-atomic snapshot in this tree taken for one log line or report. +func (c *Counts) Snapshot() Snapshot { + return Snapshot{ + Prepare: atomic.LoadInt64(&c.prepare), + Exec: atomic.LoadInt64(&c.exec), + Query: atomic.LoadInt64(&c.query), + StmtExec: atomic.LoadInt64(&c.stmtExec), + StmtQuery: atomic.LoadInt64(&c.stmtQuery), + Begin: atomic.LoadInt64(&c.begin), + Commit: atomic.LoadInt64(&c.commit), + Rollback: atomic.LoadInt64(&c.rollback), + } +} + +// Reset zeroes every counter, for reuse across benchmark iterations. +func (c *Counts) Reset() { + atomic.StoreInt64(&c.prepare, 0) + atomic.StoreInt64(&c.exec, 0) + atomic.StoreInt64(&c.query, 0) + atomic.StoreInt64(&c.stmtExec, 0) + atomic.StoreInt64(&c.stmtQuery, 0) + atomic.StoreInt64(&c.begin, 0) + atomic.StoreInt64(&c.commit, 0) + atomic.StoreInt64(&c.rollback, 0) +} + +// Total is c.Snapshot().Total(), for a caller that wants one number without +// composing the two calls. +func (c *Counts) Total() int64 { return c.Snapshot().Total() } + +// options carries WrapConnector's optional behavior. +type options struct { + latency time.Duration +} + +// Option configures WrapConnector. +type Option func(*options) + +// WithLatency injects a sleep before every counted round trip — Prepare, +// Exec, Query, a prepared statement's Exec/Query, Begin, Commit and Rollback +// alike — so a co-located backend can approximate the wall-clock cost of a +// database that is NOT co-located with the caller. It respects the caller's +// context: a canceled or expired context aborts the sleep and returns the +// context's error rather than padding out a call that is already doomed. +func WithLatency(d time.Duration) Option { + return func(o *options) { o.latency = d } +} + +// WrapConnector wraps inner so every statement-shaped call on the +// connections it hands out increments counts, and optionally sleeps first. +// inner is the driver.Connector a backend already constructs +// (mysql.NewConnector for the Dolt server backend, doltembed.NewConnector +// for the embedded backend); wrapping it rather than registering a new +// driver under a new name is what lets a test open a SECOND, counted +// connection against the SAME already-migrated embedded directory or the +// SAME live Dolt server a backend's own store already opened, so the +// production open/migrate path never has to know this wrapper exists. +func WrapConnector(inner driver.Connector, counts *Counts, opts ...Option) driver.Connector { + o := &options{} + for _, opt := range opts { + opt(o) + } + return &wrappedConnector{inner: inner, counts: counts, opts: o} +} + +type wrappedConnector struct { + inner driver.Connector + counts *Counts + opts *options +} + +func (c *wrappedConnector) Connect(ctx context.Context) (driver.Conn, error) { + inner, err := c.inner.Connect(ctx) + if err != nil { + return nil, err + } + return &wrappedConn{inner: inner, counts: c.counts, opts: c.opts}, nil +} + +func (c *wrappedConnector) Driver() driver.Driver { + return &wrappedDriver{inner: c.inner.Driver(), counts: c.counts, opts: c.opts} +} + +// wrappedDriver exists only to satisfy driver.Connector.Driver's return +// type. database/sql calls Driver() for error classification and +// sql.DB.Driver(); this wrapper never uses it to open a second connection +// outside wrappedConnector.Connect. +type wrappedDriver struct { + inner driver.Driver + counts *Counts + opts *options +} + +func (d *wrappedDriver) Open(name string) (driver.Conn, error) { + inner, err := d.inner.Open(name) + if err != nil { + return nil, err + } + return &wrappedConn{inner: inner, counts: d.counts, opts: d.opts}, nil +} + +// sleep injects the configured latency, honoring ctx cancellation. +func (o *options) sleep(ctx context.Context) error { + if o.latency <= 0 { + return nil + } + t := time.NewTimer(o.latency) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// wrappedConn wraps a driver.Conn. It implements every optional modern +// interface (ExecerContext, QueryerContext, ConnPrepareContext, +// ConnBeginTx, Pinger, SessionResetter, Validator, NamedValueChecker), +// delegating to the inner connection's implementation when present and +// falling back to the legacy path — or driver.ErrSkip — when not, so +// database/sql's behavior with the wrapper in place matches its behavior +// with the inner driver alone. +// +// NamedValueChecker is always implemented (never omitted) because +// database/sql decides parameter-conversion behavior by checking interface +// satisfaction on the WRAPPER's concrete type, not the inner driver's: +// omitting it here would silently change conversion behavior relative to +// running against the inner driver directly, regardless of what the inner +// driver itself implements. +type wrappedConn struct { + inner driver.Conn + counts *Counts + opts *options +} + +func (c *wrappedConn) Prepare(query string) (driver.Stmt, error) { + atomic.AddInt64(&c.counts.prepare, 1) + inner, err := c.inner.Prepare(query) + if err != nil { + return nil, err + } + return &wrappedStmt{inner: inner, counts: c.counts, opts: c.opts}, nil +} + +func (c *wrappedConn) Close() error { return c.inner.Close() } + +//nolint:staticcheck // legacy driver.Conn.Begin, required by the interface; BeginTx below is preferred when available. +func (c *wrappedConn) Begin() (driver.Tx, error) { + atomic.AddInt64(&c.counts.begin, 1) + inner, err := c.inner.Begin() //nolint:staticcheck + if err != nil { + return nil, err + } + return &wrappedTx{inner: inner, counts: c.counts}, nil +} + +func (c *wrappedConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + pc, ok := c.inner.(driver.ConnPrepareContext) + if !ok { + return c.Prepare(query) + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.prepare, 1) + inner, err := pc.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + return &wrappedStmt{inner: inner, counts: c.counts, opts: c.opts}, nil +} + +func (c *wrappedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + execer, ok := c.inner.(driver.ExecerContext) + if !ok { + return nil, driver.ErrSkip + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.exec, 1) + return execer.ExecContext(ctx, query, args) +} + +func (c *wrappedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := c.inner.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.query, 1) + return queryer.QueryContext(ctx, query, args) +} + +func (c *wrappedConn) BeginTx(ctx context.Context, txOpts driver.TxOptions) (driver.Tx, error) { + bt, ok := c.inner.(driver.ConnBeginTx) + if !ok { + return c.Begin() + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.begin, 1) + inner, err := bt.BeginTx(ctx, txOpts) + if err != nil { + return nil, err + } + return &wrappedTx{inner: inner, counts: c.counts}, nil +} + +func (c *wrappedConn) Ping(ctx context.Context) error { + if p, ok := c.inner.(driver.Pinger); ok { + return p.Ping(ctx) + } + return nil +} + +func (c *wrappedConn) ResetSession(ctx context.Context) error { + if r, ok := c.inner.(driver.SessionResetter); ok { + return r.ResetSession(ctx) + } + return nil +} + +func (c *wrappedConn) IsValid() bool { + if v, ok := c.inner.(driver.Validator); ok { + return v.IsValid() + } + return true +} + +func (c *wrappedConn) CheckNamedValue(nv *driver.NamedValue) error { + if chk, ok := c.inner.(driver.NamedValueChecker); ok { + return chk.CheckNamedValue(nv) + } + return driver.ErrSkip +} + +// wrappedStmt wraps a driver.Stmt the same way wrappedConn wraps a +// driver.Conn: delegate to a modern optional interface when the inner +// statement implements it, else fall back to the legacy path, and always +// implement NamedValueChecker for the reason documented on wrappedConn. +type wrappedStmt struct { + inner driver.Stmt + counts *Counts + opts *options +} + +func (s *wrappedStmt) Close() error { return s.inner.Close() } +func (s *wrappedStmt) NumInput() int { return s.inner.NumInput() } + +//nolint:staticcheck // legacy driver.Stmt.Exec, required by the interface and used as the ExecContext fallback below. +func (s *wrappedStmt) Exec(args []driver.Value) (driver.Result, error) { + atomic.AddInt64(&s.counts.stmtExec, 1) + return s.inner.Exec(args) //nolint:staticcheck +} + +//nolint:staticcheck // legacy driver.Stmt.Query, required by the interface and used as the QueryContext fallback below. +func (s *wrappedStmt) Query(args []driver.Value) (driver.Rows, error) { + atomic.AddInt64(&s.counts.stmtQuery, 1) + return s.inner.Query(args) //nolint:staticcheck +} + +func (s *wrappedStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + if err := s.opts.sleep(ctx); err != nil { + return nil, err + } + if se, ok := s.inner.(driver.StmtExecContext); ok { + atomic.AddInt64(&s.counts.stmtExec, 1) + return se.ExecContext(ctx, args) + } + dargs, err := namedToOrdinal(args) + if err != nil { + return nil, err + } + atomic.AddInt64(&s.counts.stmtExec, 1) + return s.inner.Exec(dargs) //nolint:staticcheck +} + +func (s *wrappedStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + if err := s.opts.sleep(ctx); err != nil { + return nil, err + } + if sq, ok := s.inner.(driver.StmtQueryContext); ok { + atomic.AddInt64(&s.counts.stmtQuery, 1) + return sq.QueryContext(ctx, args) + } + dargs, err := namedToOrdinal(args) + if err != nil { + return nil, err + } + atomic.AddInt64(&s.counts.stmtQuery, 1) + return s.inner.Query(dargs) //nolint:staticcheck +} + +func (s *wrappedStmt) CheckNamedValue(nv *driver.NamedValue) error { + if chk, ok := s.inner.(driver.NamedValueChecker); ok { + return chk.CheckNamedValue(nv) + } + return driver.ErrSkip +} + +// namedToOrdinal converts modern named-value args back to the legacy +// ordinal-positioned driver.Value slice Exec/Query take, for an inner +// statement that implements neither StmtExecContext nor StmtQueryContext. +// Every driver this wrapper targets (mysql, dolthub/driver) implements the +// modern interfaces, so this path is exercised only as a defensive +// fallback, never in the measured code path. +func namedToOrdinal(args []driver.NamedValue) ([]driver.Value, error) { + out := make([]driver.Value, len(args)) + for _, a := range args { + if a.Name != "" { + return nil, fmt.Errorf("sqlcount: named parameter %q not supported by the wrapped driver's legacy Exec/Query", a.Name) + } + if a.Ordinal < 1 || a.Ordinal > len(args) { + return nil, fmt.Errorf("sqlcount: parameter ordinal %d out of range for %d args", a.Ordinal, len(args)) + } + out[a.Ordinal-1] = a.Value + } + return out, nil +} + +// wrappedTx wraps a driver.Tx purely to count Commit/Rollback; it does not +// change the transaction's behavior. +type wrappedTx struct { + inner driver.Tx + counts *Counts +} + +func (t *wrappedTx) Commit() error { + atomic.AddInt64(&t.counts.commit, 1) + return t.inner.Commit() +} + +func (t *wrappedTx) Rollback() error { + atomic.AddInt64(&t.counts.rollback, 1) + return t.inner.Rollback() +} From fe54ac09ef9d60e8babe61c91bb103e7832c28b9 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Wed, 30 Sep 2026 23:22:30 +0000 Subject: [PATCH 2/5] feat(httpapi,issueops): raise batchApply envelope to 1000 items with bounded large-apply handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /v0/beads/issues:batchApply now accepts up to 1000 items (up from 100); issueops.MaxApplyBatchItems moves accordingly and the request body cap rises to 16 MiB. The whole plan still applies in exactly ONE transaction. A request over 100 items runs under a separate, flat, operator-configured whole-run budget (`bd serve --large-apply-ceiling`, 5 minutes by default) instead of the ordinary 60s request deadline; a request at or under 100 items is unaffected and keeps exactly the 60s deadline and 15s commit-retry ceiling it always had. Concurrency and shutdown are bounded, not just documented: - Oversized requests serialize through a dedicated one-wide semaphore, and at most one caller may wait for that semaphore at a time (a CAS-guarded single waiter) — a burst of oversized requests is shed with 503 rather than queuing and starving ordinary traffic behind the general semaphore. - The run deadline is stamped only once the slot is actually acquired, not at the time the request arrived. - Graceful shutdown closes new admissions before computing its drain budget, then waits out an in-flight large apply up to its own remaining deadline (plus a few seconds' grace for the response to write); any large apply still queued for the semaphore when shutdown begins is refused rather than extending the drain further. A queued caller that disconnects gets a clean 503, never a misclassified context error. `bd serve`'s orchestrator stop grace must be at least `--large-apply-ceiling` — see engdocs/SERVE_RUNBOOK.md. - ApplyBatch's commit-retry budget only extends past the ordinary 15s ceiling on this explicit large-apply path (via a context marker), with headroom reserved so the final retry attempt can't be killed mid-commit by the context's own expiry. A new `issues.batchApplyLarge` capability token on `GET /v0/beads/context` lets a client learn whether a given server accepts the raised envelope. Wall-clock measurement (embedded Dolt, whole-transaction build+commit) backs the 5-minute ceiling default: 356 items ~12.6s (23.8x headroom), 1000 items ~57.5s (5.2x headroom). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 29 + cmd/bd/serve.go | 37 +- cmd/bd/serve_test.go | 28 + engdocs/SERVE_RUNBOOK.md | 18 + internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 4 +- internal/httpapi/batch_apply.go | 50 +- internal/httpapi/large_batch_apply_test.go | 1006 +++++++++++++++++ internal/httpapi/routes.go | 36 +- internal/httpapi/server.go | 299 ++++- internal/httpapi/server_test.go | 8 +- internal/httpapi/spec/openapi.v0.yaml | 44 +- internal/storage/BUILD.bazel | 2 + internal/storage/batchfixtures/BUILD.bazel | 39 + .../storage/batchfixtures/batchfixtures.go | 21 +- internal/storage/dolt/BUILD.bazel | 3 + .../dolt/large_batch_apply_measure_test.go | 8 +- internal/storage/embeddeddolt/BUILD.bazel | 5 + .../large_batch_apply_measure_test.go | 79 +- internal/storage/sqlcount/BUILD.bazel | 21 + internal/storage/sqlcount/sqlcount.go | 7 +- internal/storage/uow/BUILD.bazel | 1 + internal/storage/uow/batch_applier.go | 66 +- .../uow/batch_applier_retry_budget_test.go | 192 ++++ issueops/batchapplier.go | 64 +- 25 files changed, 2000 insertions(+), 68 deletions(-) create mode 100644 internal/httpapi/large_batch_apply_test.go create mode 100644 internal/storage/batchfixtures/BUILD.bazel create mode 100644 internal/storage/sqlcount/BUILD.bazel create mode 100644 internal/storage/uow/batch_applier_retry_budget_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 830a122ef..34c2a3c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -596,6 +596,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`POST /v0/beads/issues:batchApply` accepts up to 1000 items, up from 100.** + `issueops.MaxApplyBatchItems` moves from 100 to 1000; the request body cap + rises from 4 MiB to 16 MiB (`maxApplyBatchBodyBytes`); and the whole plan + still applies in exactly ONE transaction — the cap raise never chunks a + request. A request over 100 items now runs under a separate, EXTENDED + whole-run budget instead of the ordinary 60s request deadline: a new + `bd serve --large-apply-ceiling` flag sets it (5 minutes by default), applied + flat regardless of item count, never scaled down. A request at or under 100 + items is completely unaffected and keeps exactly the 60s deadline it has + always had. A request over 100 items also serializes against every other + such request through a dedicated one-wide semaphore, so at most one + oversized write transaction is in flight at a time; the wait for that + semaphore is itself bounded, so a burst of oversized requests is shed with + `503` rather than queuing indefinitely and starving ordinary traffic. An + ordinary (<=100-item) request is never made to wait on it. Graceful shutdown + waits out an in-flight large apply up to its own deadline, but refuses any + large apply still queued for the semaphore once shutdown begins, rather than + extending the drain further. `bd serve`'s orchestrator stop grace must be at + least `--large-apply-ceiling`, or an external `SIGKILL` can still cut off an + in-flight large apply the drain would otherwise have waited out; see + `engdocs/SERVE_RUNBOOK.md`. A new `issues.batchApplyLarge` capability token + on `GET /v0/beads/context` lets a client learn whether a given server + accepts the raised envelope before sending a plan over the old 100-item + bound. Measured wall-clock for the whole-transaction apply (embedded Dolt, + build+commit only) backs the 5-minute default: a 356-item plan (the design + doc's primary measured shape) commits in ~12.6s (23.8x headroom under the + ceiling), and a 1000-item plan (the new cap) commits in ~57.5s (5.2x + headroom). + - **A long-running schema migration now says so instead of going quiet** ([#5997](https://github.com/gastownhall/beads/pull/5997)). Migrations are allowed to take a long time by design — migration 0047's full-table diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index 713a7ec5e..e8bbf2042 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -47,11 +47,12 @@ const ( ) var ( - serveAddr string - serveAllowNonLoopback bool - serveAuthTokenFile string - serveInsecureNoAuth bool - serveAllowedHosts []string + serveAddr string + serveAllowNonLoopback bool + serveAuthTokenFile string + serveInsecureNoAuth bool + serveAllowedHosts []string + serveLargeApplyCeiling time.Duration ) var serveCmd = &cobra.Command{ @@ -175,6 +176,8 @@ func registerServeFlags(cmd *cobra.Command) { "Serve a non-loopback bind with NO authentication. Every peer that can reach the address gets full read and claim access") cmd.Flags().StringArrayVar(&serveAllowedHosts, "allowed-host", nil, "Additional Host header value to answer to, e.g. a service DNS name. Repeatable; matched exactly, with no wildcards") + cmd.Flags().DurationVar(&serveLargeApplyCeiling, "large-apply-ceiling", 5*time.Minute, + "Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md") } // serveOptions is the part of a server's configuration that depends on NEITHER @@ -187,11 +190,12 @@ func registerServeFlags(cmd *cobra.Command) { // server configuration that cannot yet describe a server. The field names match // Config's because they become those fields verbatim, in applyTo. type serveOptions struct { - Addr string - AllowNonLoopback bool - InsecureNoAuth bool - AllowedHosts []string - Auth *httpapi.TokenFileAuth + Addr string + AllowNonLoopback bool + InsecureNoAuth bool + AllowedHosts []string + Auth *httpapi.TokenFileAuth + LargeApplyCeiling time.Duration } // applyTo writes the operator's choices onto the configuration a database arm @@ -203,6 +207,7 @@ func (o serveOptions) applyTo(cfg *httpapi.Config) { cfg.InsecureNoAuth = o.InsecureNoAuth cfg.AllowedHosts = o.AllowedHosts cfg.Auth = o.Auth + cfg.LargeApplyCeiling = o.LargeApplyCeiling } // resolveServeConfig turns the flags and their environment fallbacks into the @@ -215,14 +220,18 @@ func (o serveOptions) applyTo(cfg *httpapi.Config) { // standing in, and must not arrive after a database has been opened. func resolveServeConfig() (serveOptions, error) { cfg := serveOptions{ - Addr: serveAddr, - AllowNonLoopback: serveAllowNonLoopback, - InsecureNoAuth: serveInsecureNoAuth, - AllowedHosts: serveAllowedHosts, + Addr: serveAddr, + AllowNonLoopback: serveAllowNonLoopback, + InsecureNoAuth: serveInsecureNoAuth, + AllowedHosts: serveAllowedHosts, + LargeApplyCeiling: serveLargeApplyCeiling, } if _, err := httpapi.ValidateBindAddr(serveAddr, serveAllowNonLoopback); err != nil { return cfg, err } + if serveLargeApplyCeiling <= 0 { + return cfg, fmt.Errorf("--large-apply-ceiling must be positive, got %s", serveLargeApplyCeiling) + } tokenFile := serveAuthTokenFile if tokenFile == "" { diff --git a/cmd/bd/serve_test.go b/cmd/bd/serve_test.go index a94be1241..ea980a4d1 100644 --- a/cmd/bd/serve_test.go +++ b/cmd/bd/serve_test.go @@ -13,6 +13,7 @@ import ( "sort" "strings" "testing" + "time" "github.com/spf13/pflag" @@ -511,14 +512,17 @@ func withServeFlags(t *testing.T) { addr, nonLoopback := serveAddr, serveAllowNonLoopback token, insecure := serveAuthTokenFile, serveInsecureNoAuth hosts := serveAllowedHosts + largeApplyCeiling := serveLargeApplyCeiling t.Cleanup(func() { serveAddr, serveAllowNonLoopback = addr, nonLoopback serveAuthTokenFile, serveInsecureNoAuth = token, insecure serveAllowedHosts = hosts + serveLargeApplyCeiling = largeApplyCeiling }) serveAddr, serveAllowNonLoopback = "127.0.0.1:0", false serveAuthTokenFile, serveInsecureNoAuth = "", false serveAllowedHosts = nil + serveLargeApplyCeiling = 5 * time.Minute } func serveTokenFile(t *testing.T) string { @@ -584,6 +588,20 @@ func TestServeConfigRefusesAnUnservablePosture(t *testing.T) { apply: func(*testing.T) { serveAllowedHosts = []string{"bd.beads.svc:8080"} }, wantErr: "--allowed-host", }, + { + name: "a zero large-apply ceiling", + apply: func(*testing.T) { serveLargeApplyCeiling = 0 }, + wantErr: "--large-apply-ceiling", + }, + { + name: "a negative large-apply ceiling", + apply: func(*testing.T) { serveLargeApplyCeiling = -time.Second }, + wantErr: "--large-apply-ceiling", + }, + { + name: "a non-default large-apply ceiling", + apply: func(*testing.T) { serveLargeApplyCeiling = 90 * time.Second }, + }, } { t.Run(tc.name, func(t *testing.T) { clearServeEnv(t) @@ -647,6 +665,7 @@ func TestServeConfigCarriesTheOperatorsChoicesThrough(t *testing.T) { withServeFlags(t) serveAuthTokenFile = serveTokenFile(t) serveAllowedHosts = []string{"bd-proj.beads.svc.cluster.local", "bd-proj.beads.svc"} + serveLargeApplyCeiling = 90 * time.Second cfg, err := resolveServeConfig() if err != nil { @@ -661,6 +680,15 @@ func TestServeConfigCarriesTheOperatorsChoicesThrough(t *testing.T) { if cfg.InsecureNoAuth { t.Error("InsecureNoAuth is set without the flag") } + if cfg.LargeApplyCeiling != serveLargeApplyCeiling { + t.Errorf("LargeApplyCeiling = %s, want %s (--large-apply-ceiling did not reach the server config)", cfg.LargeApplyCeiling, serveLargeApplyCeiling) + } + + var httpCfg httpapi.Config + cfg.applyTo(&httpCfg) + if httpCfg.LargeApplyCeiling != serveLargeApplyCeiling { + t.Errorf("serveOptions.applyTo did not carry LargeApplyCeiling through: got %s, want %s", httpCfg.LargeApplyCeiling, serveLargeApplyCeiling) + } } // TestServeHelpDescribesTheAuthPosture. The help text is where an operator diff --git a/engdocs/SERVE_RUNBOOK.md b/engdocs/SERVE_RUNBOOK.md index e746571aa..5a21d9fd6 100644 --- a/engdocs/SERVE_RUNBOOK.md +++ b/engdocs/SERVE_RUNBOOK.md @@ -319,6 +319,24 @@ contexts: the budget covers a claim inside its serialization-retry budget plus its commit, because killing such a connection early would leave the client unable to tell whether its write landed. +**A large `POST /v0/beads/issues:batchApply` (over 100 items) extends this +budget rather than being cut off by it.** Such a request can legitimately still +be running at up to `--large-apply-ceiling` (5 minutes by default) when +shutdown begins, so the drain budget computed at shutdown start extends to that +request's own remaining deadline instead of the ordinary 20 seconds — the +in-flight write is waited out, not killed mid-commit. A large apply still +*queued* for the one-wide large-apply semaphore when shutdown begins is refused +with `503` immediately rather than being admitted or extending the drain +further. + +**The orchestrator's stop grace (e.g. a Kubernetes `terminationGracePeriodSeconds`) +must be at least `--large-apply-ceiling`.** The drain logic above only runs +inside this process; an external `SIGKILL` after the orchestrator's own grace +period expires cuts the process off regardless of what the drain was waiting +for, which can still sever an in-flight large apply the drain would otherwise +have waited out. Size the grace period to the ceiling, not to the ordinary 20s +drain. + Journal streams are the exception, and they have to be: a held-open response would otherwise sit through the whole budget and then be killed anyway. They get an explicit close signal when the drain starts and end on their own within a diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index ac3f37e93..c0e0be8f4 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -81,6 +81,7 @@ go_test( "events_test.go", "events_watch_test.go", "get_issue_test.go", + "large_batch_apply_test.go", "list_comments_omitted_test.go", "list_sort_test.go", "memories_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index ec5422748..9ad2dc9be 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -354,7 +354,7 @@ type ApplyBatchRequest struct { // Items The items to apply, IN THE ORDER THEY ARE TO BE APPLIED. An empty array is a `400` rather than a successful no-op: a write request that writes nothing is a client bug, and answering it cheerfully is how a client whose own plan filtered to nothing silently stops writing. // - // The 100-item cap bounds how long one request may hold a write transaction, not batch semantics. Split a larger plan; each request is atomic on its own — but splitting it changes what the end gate can see, since the gate runs over one request at a time. + // The 1000-item cap (raised from 100; see the `issues.batchApplyLarge` capability token on `ContextResponse`) bounds how long one request may hold a write transaction, not batch semantics. A request over 100 items also EXTENDS the effective request deadline: the server grants a flat, operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) instead of the ordinary small-request deadline, so a plan that genuinely needs minutes to apply is not cut off mid-commit. It never narrows: a request at or under 100 items keeps exactly the deadline it has always had. Split a larger plan if you must; each request is atomic on its own — but splitting it changes what the end gate can see, since the gate runs over one request at a time, and the server itself never splits a plan you send it: a single `items` array either commits in full or changes nothing. // // A per-item refusal names its offender as `items[i].kind.member`. Items []ApplyItem `json:"items"` @@ -1010,7 +1010,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the one behavior token is `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, and `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 2da39156b..21b40f174 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -37,11 +37,37 @@ const ( // It bounds how long one request may hold a write transaction — not batch // semantics, which have no size in them. maxApplyBatchItems = issueops.MaxApplyBatchItems - // maxApplyBatchBodyBytes bounds the request body. A hundred items each + // maxApplyBatchBodyBytes bounds the request body. A thousand items each // carrying a description, a design, acceptance criteria and a metadata // document is the shape this has to admit, so it refuses the absurd before - // any of it is parsed. It is the batch create's bound for the same reason. - maxApplyBatchBodyBytes = 4 << 20 + // any of it is parsed. + // + // Raised from 4 MiB to 16 MiB alongside maxApplyBatchItems's 100->1000 + // raise: the measured max shape's body (102 creates, descriptions + // totaling about 878 KB) is about 1.1 MB, and 1000 items at the same + // density is comfortably inside 16 MiB with headroom for a caller that + // writes longer descriptions than that sample. + // + // See TestCapBatchApplyLargeTiesAllThreeLimits: this constant, + // largeApplyItemThreshold and issueops.MaxApplyBatchItems are pinned + // together with CapBatchApplyLarge, the token that advertises all three. + // + // See maxInflight's doc comment (server.go) for the worst-case concurrent + // decode memory this implies: maxInflight requests may each be decoding a + // body up to this size at once, before the large-apply semaphore narrows + // how many of them can go on to actually run. + maxApplyBatchBodyBytes = 16 << 20 + // largeApplyItemThreshold is where a request crosses from "the original, + // always-supported shape" to "the raised envelope": a request with MORE + // than this many items holds the largeApplySem slot (serializing + // oversized transactions one at a time, Server.acquireLargeApply) and + // runs under a whole separate, EXTENDED budget (Server.largeApplyCeiling) + // instead of the ordinary requestDeadline every other request gets. It is + // deliberately the OLD cap, not some fraction of the new one: every + // request at or under this threshold runs under EXACTLY the deadline it + // always has — route() applies requestDeadline unconditionally, and + // nothing in this file touches it for such a request. + largeApplyItemThreshold = 100 ) // The document's member list at each of this body's levels. Every schema is @@ -118,6 +144,24 @@ func (s *Server) handleApplyBatch(w http.ResponseWriter, r *http.Request) { return } + // One-wide "large write" semaphore: a request carrying more than + // largeApplyItemThreshold items serializes against every other large + // apply, so at most one oversized transaction holds a write connection + // at a time, and runs under an EXTENDED budget built fresh at the moment + // it is admitted (Server.acquireLargeApply). An ordinary (<=threshold) + // request never touches this, is never waited on by one, and keeps + // r.Context() exactly as route() built it — requestDeadline, + // unconditionally, the same as it always has. + if len(request.Items) > largeApplyItemThreshold { + runCtx, release, err := s.acquireLargeApply(r.Context()) + if err != nil { + s.failApplyBatch(w, r, request, err) + return + } + defer release() + r = r.WithContext(runCtx) + } + applier, err := s.batchApplier(r) if err != nil { s.failApplyBatch(w, r, request, err) diff --git a/internal/httpapi/large_batch_apply_test.go b/internal/httpapi/large_batch_apply_test.go new file mode 100644 index 000000000..e80c4cec4 --- /dev/null +++ b/internal/httpapi/large_batch_apply_test.go @@ -0,0 +1,1006 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/issueops" +) + +// The pins for the raised POST issues:batchApply envelope: the 16 MiB body +// cap, the one-wide large-write semaphore that serializes oversized +// transactions without touching an ordinary request, and the EXTENDED (never +// narrowed) run budget a request over largeApplyItemThreshold items gets once +// it is admitted. The item-count cap itself (issueops.MaxApplyBatchItems, +// 100->1000) is pinned at the role and storage layers (issueops, +// internal/storage/uow, internal/storage/embeddeddolt, internal/storage/dolt); +// backend/conformance's RunBatchApplyBoundsTheItemCount already scales to it +// automatically, which is why it is not repinned here. + +// TestApplyBatchBodyCapIsEnforcedWhileReading mirrors +// TestClaimBodyCapIsEnforcedWhileReading (claim_test.go): the raised 16 MiB +// cap is refused mid-read, at the decoder, before any member exists to name. +func TestApplyBatchBodyCapIsEnforcedWhileReading(t *testing.T) { + oversized := `{"actor":"` + strings.Repeat("x", maxApplyBatchBodyBytes) + `"}` + r := httptest.NewRequest(http.MethodPost, batchApplyPath, strings.NewReader(oversized)) + + members, res := decodeJSONObject(httptest.NewRecorder(), r, maxApplyBatchBodyBytes) + if res == nil { + t.Fatalf("a %d-byte body was accepted (%d members)", len(oversized), len(members)) + } + if res.Problem.Status != http.StatusBadRequest || res.Problem.Code != string(CodeInvalidArgument) { + t.Errorf("problem = %d/%s, want 400/%s", res.Problem.Status, res.Problem.Code, CodeInvalidArgument) + } + if res.Problem.Detail == nil || !strings.Contains(*res.Problem.Detail, "larger than") { + t.Errorf("detail = %v, want it to say the body was too large", res.Problem.Detail) + } +} + +// TestApplyBatchAcceptsABodyThePreB1CapWouldHaveRefused proves the raise is +// real end-to-end over the wire, not just in the constant: a body north of the +// OLD 4 MiB cap (but comfortably under the new 16 MiB one) must now succeed, +// built from ordinary create items rather than one oversized member. +func TestApplyBatchAcceptsABodyThePreB1CapWouldHaveRefused(t *testing.T) { + const oldCap = 4 << 20 + applier := &roleBatchApplier{} + ts := newApplyBatchServer(t, applier) + + // 60 items with a ~100 KiB description each is ~6 MiB of body — over the + // old 4 MiB cap, under the new 16 MiB one. + desc := strings.Repeat("d", 100<<10) + items := make([]string, 60) + for i := range items { + items[i] = fmt.Sprintf(`{"kind":"create","create":{"title":"t-%d","description":%q}}`, i, desc) + } + body := `{"actor":"alice","items":[` + strings.Join(items, ",") + `]}` + if len(body) <= oldCap { + t.Fatalf("test body is %d bytes, want more than the old %d-byte cap to actually exercise the raise", len(body), oldCap) + } + if int64(len(body)) >= maxApplyBatchBodyBytes { + t.Fatalf("test body is %d bytes, want less than the new %d-byte cap", len(body), maxApplyBatchBodyBytes) + } + + resp := ts.claim(t, batchApplyPath, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 for a %d-byte body under the raised cap: %s", resp.StatusCode, len(body), readAll(t, resp)) + } +} + +// TestCapBatchApplyLargeTiesAllThreeLimits pins the three numbers +// CapBatchApplyLarge (issues.batchApplyLarge) promises together: an older +// client that checks this token before dialing an over-100-item plan relies +// on ALL THREE — the item cap, the body cap and the run-budget ceiling — so a +// change that lowers one of them without also removing the token would +// advertise an envelope this server no longer actually honors. +func TestCapBatchApplyLargeTiesAllThreeLimits(t *testing.T) { + const ( + wantMaxItems = 1000 + wantMaxBodyBytes = 16 << 20 + wantThreshold = 100 + wantDefaultCeiling = 5 * time.Minute + ) + if issueops.MaxApplyBatchItems != wantMaxItems { + t.Errorf("issueops.MaxApplyBatchItems = %d, want %d", issueops.MaxApplyBatchItems, wantMaxItems) + } + if maxApplyBatchBodyBytes != wantMaxBodyBytes { + t.Errorf("maxApplyBatchBodyBytes = %d, want %d", maxApplyBatchBodyBytes, wantMaxBodyBytes) + } + if largeApplyItemThreshold != wantThreshold { + t.Errorf("largeApplyItemThreshold = %d, want %d", largeApplyItemThreshold, wantThreshold) + } + if defaultLargeApplyCeiling != wantDefaultCeiling { + t.Errorf("defaultLargeApplyCeiling = %s, want %s", defaultLargeApplyCeiling, wantDefaultCeiling) + } + if !slices.Contains(behaviorCapabilities, CapBatchApplyLarge) { + t.Fatal("CapBatchApplyLarge is not advertised in behaviorCapabilities; a client cannot discover this envelope") + } +} + +// TestAcquireLargeApplySerializesOneAtATime drives acquireLargeApply +// directly: a second caller blocks until the first releases. +func TestAcquireLargeApplySerializesOneAtATime(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, // generous: this test is about serialization, not timeout + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + + runCtx1, release1, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("first acquireLargeApply: %v", err) + } + dl1, ok := runCtx1.Deadline() + if !ok { + t.Fatal("first run context carries no deadline") + } + if got := s.largeApplyDeadline.Load(); got == nil || !got.Equal(dl1) { + t.Fatalf("largeApplyDeadline while held = %v, want %v", got, dl1) + } + + // A second, live caller blocks until the first releases. + second := make(chan struct{}) + go func() { + _, release2, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Errorf("second acquireLargeApply: %v", err) + return + } + release2() + close(second) + }() + + select { + case <-second: + t.Fatal("the second acquire completed before the first released — the slot is not one-wide") + case <-time.After(50 * time.Millisecond): + } + + release1() + if got := s.largeApplyDeadline.Load(); got != nil { + t.Fatalf("largeApplyDeadline after release = %v, want nil", got) + } + + select { + case <-second: + case <-time.After(2 * time.Second): + t.Fatal("the second acquire never completed after the first released") + } +} + +// TestAcquireLargeApplyDeadlineStartsAtAcquisitionNotAtCall pins that the run +// budget is measured from the moment the slot is actually taken, not from +// when the caller first started waiting for it: a caller queued for a while +// must still get the FULL largeApplyCeiling once admitted, not whatever is +// left of a deadline that started ticking back when it first queued. +func TestAcquireLargeApplyDeadlineStartsAtAcquisitionNotAtCall(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + _, release1, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("first acquireLargeApply: %v", err) + } + + // The queue delay must be LARGE relative to the slack below, not the + // other way around: a mutation that measures the run deadline from the + // CALL (when acquireLargeApply was entered and started queueing) instead + // of from ACQUISITION (when the slot was actually won) only differs from + // the correct behavior by roughly the queue delay. A short queue delay + // (e.g. 250ms) against generous slack (e.g. 2s) would hide that + // difference entirely; a 2s queue delay against 250ms slack cannot. + const queueDelay = 2 * time.Second + const slack = 250 * time.Millisecond + queueStart := time.Now() + go func() { + time.Sleep(queueDelay) + release1() + }() + + runCtx2, release2, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("second acquireLargeApply: %v", err) + } + defer release2() + acquiredAt := time.Now() + + if waited := acquiredAt.Sub(queueStart); waited < queueDelay/2 { + t.Fatalf("test setup: the second caller was not actually queued (waited only %s)", waited) + } + dl2, ok := runCtx2.Deadline() + if !ok { + t.Fatal("second run context carries no deadline") + } + budget := dl2.Sub(acquiredAt) + if budget < s.largeApplyCeiling-slack || budget > s.largeApplyCeiling+slack { + t.Errorf("run budget measured from ACQUISITION = %s, want ~%s (the deadline must start when the lock is acquired, not when the wait began)", budget, s.largeApplyCeiling) + } +} + +// TestAcquireLargeApplyRefusesAfterClosing pins the drain-safety half of +// acquireLargeApply: once s.closing fires, every NEW acquisition attempt is +// refused with ErrBusy rather than being admitted or left to hang — which is +// what lets Serve's graceful-drain budget be computed ONCE at shutdown start +// and still be correct (see Serve's doc comment). +func TestAcquireLargeApplyRefusesAfterClosing(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + close(s.closing) + + if _, _, err := s.acquireLargeApply(context.Background()); !errors.Is(err, ErrBusy) { + t.Fatalf("acquireLargeApply after closing = %v, want ErrBusy", err) + } + + // The slot itself must stay free: a refusal here is not a leak. + select { + case s.largeApplySem <- struct{}{}: + default: + t.Fatal("largeApplySem appears held after a refusal that never admitted anything") + } +} + +// TestAdmitLargeApplyRefusesIfClosingRacesInAfterTheSlotIsAlreadyWon pins item +// 3's other half directly: a select with multiple ready cases (largeApplySem +// and s.closing both ready in acquireLargeApply) picks pseudo-randomly, so a +// goroutine can win the large-apply slot in the very instant a graceful +// shutdown begins — after Serve has already snapshotted the drain budget. +// admitLargeApply's post-store recheck of s.closing is what refuses that +// admission instead of letting it run under a stale budget. This test +// exercises admitLargeApply directly, simulating that exact ordering +// (slot already taken, THEN closing fires, THEN admitLargeApply runs) +// without depending on actually winning a timing race. +func TestAdmitLargeApplyRefusesIfClosingRacesInAfterTheSlotIsAlreadyWon(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + s.largeApplySem <- struct{}{} // simulate acquireLargeApply's select already having won the slot + close(s.closing) // shutdown raced in between winning the slot and admission + + runCtx, release, err := s.admitLargeApply(context.Background()) + if !errors.Is(err, ErrBusy) { + t.Fatalf("admitLargeApply after a same-instant close = %v, want ErrBusy", err) + } + if runCtx != nil || release != nil { + t.Fatalf("admitLargeApply after a same-instant close returned runCtx=%v release non-nil=%t, want both nil", runCtx, release != nil) + } + if got := s.largeApplyDeadline.Load(); got != nil { + t.Fatalf("largeApplyDeadline left stamped at %v after a refused admission, want nil", *got) + } + + // The slot must be released back, not leaked, by the refusal. + select { + case s.largeApplySem <- struct{}{}: + default: + t.Fatal("largeApplySem appears held after admitLargeApply refused a same-instant-close admission") + } +} + +// TestLargeApplyRefusalIsBusyNotInternal is the HTTP-level twin of +// TestAcquireLargeApplyRefusesAfterClosing's sibling case: a request that +// gives up while queued for the large-apply slot — because the bounded wait +// (semTimeout) expired — must answer 503 busy with Retry-After, never the +// non-retryable 500 a bare context error would produce. Nothing was written, +// so nothing here is unsafe to retry. +func TestLargeApplyRefusalIsBusyNotInternal(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + defer close(applier.gate) + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.semTimeout = 100 * time.Millisecond + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + resp := ts.claim(t, batchApplyPath, bigBody) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 busy: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeBusy) { + t.Errorf("code = %v, want %s", body["code"], CodeBusy) + } + if ra := resp.Header.Get("Retry-After"); ra == "" { + t.Error("missing Retry-After header on a busy refusal") + } +} + +// TestAcquireLargeApplyRefusesCleanlyWhenAQueuedCallerDisconnects is item +// 7's explicit disconnect sub-bullet: a request queued for the large-apply +// slot whose OWN context ends first — the client hung up, or its ordinary +// (unrelated) request deadline expired — must answer the same ErrBusy every +// other refusal above answers. Nothing was written on this path, so it is +// safe to retry; returning the raw ctx.Err() instead would let it fall +// through to a non-retryable 500, misrepresenting a refusal that cost +// nothing as a server fault. +func TestAcquireLargeApplyRefusesCleanlyWhenAQueuedCallerDisconnects(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, // long enough that ctx, not the timer, ends the wait + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + _, release1, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("first acquireLargeApply: %v", err) + } + defer release1() + + callerCtx, cancel := context.WithCancel(context.Background()) + queuedErr := make(chan error, 1) + go func() { + _, _, err := s.acquireLargeApply(callerCtx) + queuedErr <- err + }() + + // Give the queued caller time to actually reach the bounded wait before + // disconnecting it, so this exercises the ctx.Done() branch, not a race + // against acquireLargeApply's own setup. + time.Sleep(50 * time.Millisecond) + cancel() // the client hangs up (or its own unrelated deadline expires) + + select { + case err := <-queuedErr: + if !errors.Is(err, ErrBusy) { + t.Errorf("queued caller's error after its own ctx ended = %v, want ErrBusy (never a raw ctx.Err() misclassified as a 500)", err) + } + if errors.Is(err, context.Canceled) { + t.Errorf("queued caller's error = %v wraps context.Canceled directly; it must be exactly ErrBusy, not a ctx error a caller could mistake for its own cancellation", err) + } + case <-time.After(2 * time.Second): + t.Fatal("acquireLargeApply never returned after the queued caller's context was canceled") + } +} + +// TestLargeApplyQueueDoesNotStarveOrdinaryRequests reproduces the DoS this +// slice fixes: route() acquires the GENERAL sem slot before the handler ever +// runs and holds it for the handler's whole lifetime, so a goroutine that +// then blocks UNBOUNDED on the one-wide large-apply lock starves every other +// request behind a saturated sem. Two requests queued behind a held +// large-apply slot must each give up (and free their sem slot) within a +// bounded wait, so a later ordinary request is still served promptly rather +// than finding the general sem exhausted by requests parked forever. +func TestLargeApplyQueueDoesNotStarveOrdinaryRequests(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + defer close(applier.gate) + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.sem = make(chan struct{}, 2) // a small general pool, easy to saturate + s.semTimeout = 200 * time.Millisecond + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + // Saturate the tuned general sem with requests that will each queue + // behind the held large-apply slot. + queuedDone := make(chan claimResult, 2) + for i := 0; i < 2; i++ { + go func() { queuedDone <- ts.claimAsync(bigBody) }() + } + + // Each queued request must be SHED — 503 busy — well inside a few + // multiples of the tuned semTimeout, not held open indefinitely. Before + // the fix (an unbounded select on ctx.Done() alone) this never returns, + // and the loop below times out instead — the "failing before, passing + // after" reproduction the fix is checked against. + for i := 0; i < 2; i++ { + select { + case res := <-queuedDone: + if res.err != nil { + t.Fatalf("queued large request %d transport error: %v", i, res.err) + } + if res.resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("queued large request %d status = %d, want 503 busy: %s", i, res.resp.StatusCode, readAll(t, res.resp)) + } else if ra := res.resp.Header.Get("Retry-After"); ra == "" { + t.Errorf("queued large request %d carries no Retry-After header", i) + } + case <-time.After(2 * time.Second): + t.Fatalf("queued large request %d never gave up the general sem slot it held while parked on the large-apply lock — this is the starvation bug", i) + } + } + + // Both queued requests gave their sem slots back. A fresh, ordinary small + // request must be served promptly — the general sem must not still be + // exhausted by requests that gave up on the large-apply lock but never + // released it. + start := time.Now() + smallResp := ts.claim(t, batchApplyPath, createItemsBody("carol", 1)) + if smallResp.StatusCode != http.StatusOK { + t.Fatalf("small request after the queue cleared status = %d, want 200: %s", smallResp.StatusCode, readAll(t, smallResp)) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("small request after the queue cleared took %s, want well under a second", elapsed) + } +} + +// TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout is item 1 of the +// coordinator's second review: a BOUNDED wait for largeApplySem is not +// enough on its own. Every large request that queues for the slot still +// holds its OWN general sem slot for the whole wait (route() acquires sem +// before the handler runs), so a burst of large requests could occupy a +// semTimeout's worth of general concurrency slots merely queueing — even +// though only one of them could ever win the large-apply slot — starving +// small requests behind a saturated sem for that entire window. +// +// The fix caps the number of WAITERS at one: everything beyond the request +// already holding largeApplySem and the one already queued behind it is +// refused immediately (ErrBusy, no wait, no sem slot held for it). This test +// pins the OBSERABLE consequence directly: with a small general sem pool and +// a semTimeout long enough that the old bounded-but-unbounded-waiter-count +// behavior would visibly stall small requests, 16 concurrent large requests +// racing 8 concurrent small ones must never make a small request wait on the +// large-apply contention at all — each small one completes in well under +// 100ms, not "eventually, once a wait times out". +func TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + gateClosed := false + defer func() { + if !gateClosed { + close(applier.gate) + } + }() + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.sem = make(chan struct{}, 4) // small general pool: easy to saturate if waiters pile up + s.semTimeout = 5 * time.Second // long enough that an old-style multi-waiter stall would be obvious + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + + const numBig = 16 + bigDone := make(chan claimResult, numBig) + for i := 0; i < numBig; i++ { + go func() { bigDone <- ts.claimAsync(bigBody) }() + } + // Let the first big request actually take the slot and start running + // (blocked on the gate) before firing the small requests, so the small + // requests race real large-apply contention, not just a cold start. + waitForCallCount(t, applier, 1) + + const numSmall = 8 + type smallResult struct { + claimResult + elapsed time.Duration + } + smallDone := make(chan smallResult, numSmall) + for i := 0; i < numSmall; i++ { + go func(i int) { + start := time.Now() + res := ts.claimAsync(createItemsBody(fmt.Sprintf("small-%d", i), 1)) + smallDone <- smallResult{claimResult: res, elapsed: time.Since(start)} + }(i) + } + + for i := 0; i < numSmall; i++ { + select { + case res := <-smallDone: + if res.err != nil { + t.Fatalf("small request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("small request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + if res.elapsed > 100*time.Millisecond { + t.Errorf("small request took %s, want well under 100ms: a waiter for the large-apply slot must never hold a general sem slot long enough to stall a small request", res.elapsed) + } + case <-time.After(2 * time.Second): + t.Fatalf("small request %d never completed — this is the starvation bug item 1 fixes", i) + } + } + + // Drain every big request so the test can exit cleanly: exactly one runs + // (blocked on the gate) at a time, at most one more ever queues and then + // wins the slot once the first releases, and the rest are refused + // immediately. Closing (rather than sending once) unblocks every request + // that is, or will be, parked on the gate, however many turns it takes + // the single-waiter slot to cycle through. + close(applier.gate) + gateClosed = true + for i := 0; i < numBig; i++ { + select { + case <-bigDone: + case <-time.After(3 * time.Second): + t.Fatalf("big request %d never returned", i) + } + } +} + +// TestDrainWaitsForInFlightLargeApplyButRefusesAQueuedOne is the graceful- +// shutdown story: a large apply already holding the slot when shutdown +// begins is waited out and its commit lands; a second one still queued for +// the slot when s.closing fires is refused (503 busy) rather than extending +// the drain further or being force-killed. +func TestDrainWaitsForInFlightLargeApplyButRefusesAQueuedOne(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + stdout := &strings.Builder{} + stderr := &lockedBuffer{} + srv, err := Listen(rolesConfig(Config{ + BatchApplier: applier, + Addr: "127.0.0.1:0", + Stdout: stdout, + Stderr: stderr, + LargeApplyCeiling: time.Minute, + })) + if err != nil { + t.Fatalf("Listen: %v", err) + } + // Long enough that s.closing, not the timer, is what refuses the queued + // request below. + srv.semTimeout = 5 * time.Second + + ctx, cancel := context.WithCancel(context.Background()) + serveDone := make(chan error, 1) + go func() { serveDone <- srv.Serve(ctx) }() + + ts := &testServer{Server: srv, client: &http.Client{Timeout: 20 * time.Second}, base: "http://" + srv.Addr()} + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + queuedDone := make(chan claimResult, 1) + go func() { queuedDone <- ts.claimAsync(bigBody) }() + // Give the queued request time to reach acquireLargeApply's bounded wait + // before shutdown begins; decoding a small fixed body is fast enough that + // a short fixed sleep is the simplest synchronization here, matching this + // file's other fixed-delay waits. + time.Sleep(100 * time.Millisecond) + + cancel() // begin graceful shutdown with the first in flight, the second queued + + select { + case res := <-queuedDone: + if res.err != nil { + t.Fatalf("queued large request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("queued request during drain status = %d, want 503 busy: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(3 * time.Second): + t.Fatal("the queued large request was not refused once shutdown began") + } + + // The in-flight one must be waited out, not cut off: let it finish now. + close(applier.gate) + + select { + case res := <-holderDone: + if res.err != nil { + t.Fatalf("in-flight large request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("in-flight large request status during drain = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(5 * time.Second): + t.Fatal("the in-flight large request never completed during drain") + } + + select { + case err := <-serveDone: + if err != nil { + t.Fatalf("Serve: %v", err) + } + case <-time.After(30 * time.Second): + t.Fatal("Serve did not return after shutdown") + } + + log := stderr.String() + if !strings.Contains(log, "event=shutdown_complete") { + t.Errorf("shutdown did not complete cleanly:\n%s", log) + } + if strings.Contains(log, "event=shutdown_forced") { + t.Errorf("shutdown was forced rather than completing the drain:\n%s", log) + } + if got := applier.callCount(); got != 1 { + t.Errorf("ApplyBatch call count = %d, want 1 (the queued request must never have reached the role)", got) + } +} + +// TestServeExtendsDrainPastAFixedFloorForAnInFlightLargeApply is item 3 of +// the coordinator's second review: TestDrainWaitsForInFlightLargeApplyButRefusesAQueuedOne +// above exercises the real Serve() path, but the in-flight large apply there +// is released within a couple hundred milliseconds of shutdown starting — far +// under even the OLD fixed drainTimeout constant (20s) — so a mutation +// reverting Serve to always drain for a fixed floor, ignoring +// drainBudget's extension for an in-flight large apply's own remaining +// deadline, would still pass it undetected. +// +// This test shrinks the floor (Server.drainTimeout) to something a unit test +// can afford to actually exceed, holds a large apply past that shrunk floor +// but still well under its own extended deadline, and asserts Serve waits it +// out rather than force-killing the connection: proof the extension is wired +// into Serve's real drain, not just correct in the drainBudget helper. +func TestServeExtendsDrainPastAFixedFloorForAnInFlightLargeApply(t *testing.T) { + const shrunkFloor = 150 * time.Millisecond + const heldPastFloor = 600 * time.Millisecond // several floors, still << the ceiling below + const ceiling = 5 * time.Second + + applier := &controlledApplier{gate: make(chan struct{})} + stdout := &strings.Builder{} + stderr := &lockedBuffer{} + srv, err := Listen(rolesConfig(Config{ + BatchApplier: applier, + Addr: "127.0.0.1:0", + Stdout: stdout, + Stderr: stderr, + LargeApplyCeiling: ceiling, + })) + if err != nil { + t.Fatalf("Listen: %v", err) + } + srv.drainTimeout = shrunkFloor + + ctx, cancel := context.WithCancel(context.Background()) + serveDone := make(chan error, 1) + go func() { serveDone <- srv.Serve(ctx) }() + + ts := &testServer{Server: srv, client: &http.Client{Timeout: 20 * time.Second}, base: "http://" + srv.Addr()} + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + cancel() // begin graceful shutdown with the large apply in flight + + // Release the gate well past shrunkFloor. If Serve used the fixed floor + // instead of drainBudget's extension, Shutdown's drainCtx would have + // expired by now and forced the connection closed already. + time.Sleep(heldPastFloor) + close(applier.gate) + + select { + case res := <-holderDone: + if res.err != nil { + t.Fatalf("in-flight large request transport error (a fixed-floor drain would force-kill this connection before releasing the gate above): %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("in-flight large request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(5 * time.Second): + t.Fatal("the in-flight large request never completed during drain") + } + + select { + case err := <-serveDone: + if err != nil { + t.Fatalf("Serve: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Serve did not return after shutdown") + } + + log := stderr.String() + if !strings.Contains(log, "event=shutdown_complete") { + t.Errorf("shutdown did not complete cleanly:\n%s", log) + } + if strings.Contains(log, "event=shutdown_forced") { + t.Errorf("shutdown was forced rather than extending the drain past the shrunk floor for the in-flight large apply:\n%s", log) + } +} + +// TestDrainBudget pins drainBudget's extension behavior directly, without +// paying for a real multi-second wait: no large apply in flight drains in +// exactly drainTimeout; one in flight with a deadline further out than +// drainTimeout EXTENDS the budget to that deadline; one with a deadline +// closer than drainTimeout (e.g. it is nearly done) never SHRINKS the budget +// below drainTimeout. +func TestDrainBudget(t *testing.T) { + farFuture := time.Now().Add(time.Hour) + nearFuture := time.Now().Add(time.Second) + + cases := []struct { + name string + dl *time.Time + want time.Duration + }{ + {"no large apply in flight", nil, drainTimeout}, + {"large apply deadline far past drainTimeout", &farFuture, time.Until(farFuture) + drainGrace}, + {"large apply deadline closer than drainTimeout", &nearFuture, drainTimeout}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := drainBudget(drainTimeout, tc.dl) + const slack = 2 * time.Second + if got < tc.want-slack || got > tc.want+slack { + t.Errorf("drainBudget(%v) = %s, want ~%s", tc.dl, got, tc.want) + } + }) + } +} + +// deadlineCapturingApplier records the ctx.Deadline() each call actually +// received, so an HTTP-level test can check what budget handleApplyBatch +// handed the role without a real backend or a sleep-based race. +type deadlineCapturingApplier struct { + mu sync.Mutex + deadlines []time.Time +} + +func (d *deadlineCapturingApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (issueops.ApplyBatchResult, error) { + dl, _ := ctx.Deadline() + d.mu.Lock() + d.deadlines = append(d.deadlines, dl) + d.mu.Unlock() + return issueops.ApplyBatchResult{Items: make([]issueops.ItemResult, len(req.Items))}, nil +} + +func (d *deadlineCapturingApplier) last(t *testing.T) time.Time { + t.Helper() + d.mu.Lock() + defer d.mu.Unlock() + if len(d.deadlines) == 0 { + t.Fatal("ApplyBatch was never called") + } + return d.deadlines[len(d.deadlines)-1] +} + +// TestApplyBatchDeadlineAtItemCounts pins which budget governs the context +// the role actually receives at each side of largeApplyItemThreshold: EXACTLY +// requestDeadline (60s) at and under the threshold — route()'s own +// unconditional deadline, untouched by anything in this file — and EXACTLY +// the configured largeApplyCeiling above it, never a value scaled by item +// count. A distinctive, non-default ceiling is configured so a test passing +// by coincidence against a hardcoded production constant is not possible. +func TestApplyBatchDeadlineAtItemCounts(t *testing.T) { + const testCeiling = 17 * time.Second + applier := &deadlineCapturingApplier{} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.largeApplyCeiling = testCeiling + }) + + cases := []struct { + name string + items int + want time.Duration + }{ + {"at threshold (100)", largeApplyItemThreshold, requestDeadline}, + {"just over threshold (101)", largeApplyItemThreshold + 1, testCeiling}, + {"500", 500, testCeiling}, + {"at cap (1000)", issueops.MaxApplyBatchItems, testCeiling}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + start := time.Now() + resp := ts.claim(t, batchApplyPath, createItemsBody("alice", tc.items)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := applier.last(t).Sub(start) + const slack = 5 * time.Second + if got < tc.want-slack || got > tc.want+slack { + t.Errorf("%d items: applier's ctx budget = %s, want ~%s", tc.items, got, tc.want) + } + }) + } +} + +// TestApplyBatchDeadlineExceedsTheOrdinaryRequestDeadlineWhenConfiguredTo is +// item 7's large-run-ctx sub-bullet: TestApplyBatchDeadlineAtItemCounts above +// configures testCeiling = 17s, which is BELOW requestDeadline (60s) — so a +// mutation that narrows the large-apply run context to +// min(largeApplyCeiling, requestDeadline), or otherwise lets route()'s +// ordinary 60s deadline leak into the detached run context, would produce the +// exact same 17s result and pass undetected. A ceiling ABOVE requestDeadline +// is the only configuration that can tell the two apart, so this test uses +// one (97s). +func TestApplyBatchDeadlineExceedsTheOrdinaryRequestDeadlineWhenConfiguredTo(t *testing.T) { + const testCeiling = 97 * time.Second + if testCeiling <= requestDeadline { + t.Fatalf("test setup: testCeiling (%s) must exceed requestDeadline (%s) to distinguish an extended budget from a narrowed one", testCeiling, requestDeadline) + } + applier := &deadlineCapturingApplier{} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.largeApplyCeiling = testCeiling + }) + + start := time.Now() + resp := ts.claim(t, batchApplyPath, createItemsBody("alice", largeApplyItemThreshold+1)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := applier.last(t).Sub(start) + const slack = 5 * time.Second + if got < testCeiling-slack { + t.Errorf("applier's ctx budget = %s, want ~%s: the large apply's own ceiling must not be narrowed to the ordinary %s request deadline", got, testCeiling, requestDeadline) + } +} + +// controlledApplier is a BatchApplier that blocks a call carrying more than +// largeApplyItemThreshold items on a caller-controlled gate, so an HTTP-level +// test can prove the semaphore serializes oversized requests without a real +// backend or a sleep-based race. +type controlledApplier struct { + mu sync.Mutex + calls []int // len(req.Items), one entry per call, in arrival order + + gate chan struct{} // closed to release every big call waiting on it +} + +func (c *controlledApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (issueops.ApplyBatchResult, error) { + c.mu.Lock() + c.calls = append(c.calls, len(req.Items)) + c.mu.Unlock() + if len(req.Items) > largeApplyItemThreshold { + select { + case <-c.gate: + case <-ctx.Done(): + return issueops.ApplyBatchResult{}, ctx.Err() + } + } + return issueops.ApplyBatchResult{Items: make([]issueops.ItemResult, len(req.Items))}, nil +} + +func (c *controlledApplier) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.calls) +} + +// createItemsBody builds a minimal valid ApplyBatchRequest body with n create +// items, for a test that only cares about item COUNT, not content. +func createItemsBody(actor string, n int) string { + items := make([]string, n) + for i := range items { + items[i] = fmt.Sprintf(`{"kind":"create","create":{"title":"t-%d"}}`, i) + } + return fmt.Sprintf(`{"actor":%q,"items":[%s]}`, actor, strings.Join(items, ",")) +} + +// TestLargeApplySemaphoreBoundaryIsInclusiveOfThreshold pins the exact +// largeApplyItemThreshold boundary: a request carrying EXACTLY the threshold +// must never touch largeApplySem at all, even while another request holds it +// — it must complete immediately rather than queuing. One item more must +// queue behind it. This is the `>` (not `>=`) comparison in handleApplyBatch, +// pinned directly: mutating it to `>=` makes the boundary case above block +// and fail this test. +func TestLargeApplySemaphoreBoundaryIsInclusiveOfThreshold(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.semTimeout = 200 * time.Millisecond + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + exactBody := createItemsBody("bob", largeApplyItemThreshold) + start := time.Now() + resp := ts.claim(t, batchApplyPath, exactBody) + elapsed := time.Since(start) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + if elapsed > 100*time.Millisecond { + t.Errorf("a %d-item request took %s while the large-apply slot was held; it must not wait on it", largeApplyItemThreshold, elapsed) + } + + close(applier.gate) + res := <-holderDone + if res.err != nil { + t.Fatalf("large request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Fatalf("large request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } +} + +// TestLargeApplyRequestsSerializeOverHTTP is the end-to-end version of +// TestAcquireLargeApplySerializesOneAtATime: two concurrent requests each +// carrying more than largeApplyItemThreshold items must not run inside +// ApplyBatch at the same time, while a concurrent SMALL (<=threshold) request +// is never made to wait behind either of them. +func TestLargeApplyRequestsSerializeOverHTTP(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier})) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + smallBody := createItemsBody("bob", 5) + + firstDone := make(chan claimResult, 1) + go func() { firstDone <- ts.claimAsync(bigBody) }() + + // Wait for the first big request to actually enter ApplyBatch (holding + // the semaphore) before starting the second. + waitForCallCount(t, applier, 1) + + secondDone := make(chan claimResult, 1) + go func() { secondDone <- ts.claimAsync(bigBody) }() + + // The second big request must NOT reach ApplyBatch while the first still + // holds the slot: it blocks in acquireLargeApply instead. + select { + case res := <-secondDone: + t.Fatalf("the second large request completed before the first (status=%v err=%v) — the large-write semaphore did not serialize them", statusOf(res), res.err) + case <-time.After(50 * time.Millisecond): + } + if got := applier.callCount(); got != 1 { + t.Fatalf("ApplyBatch call count while the first large request is held = %d, want 1: the second must not have reached the role yet", got) + } + + // A concurrent SMALL request is unaffected by the in-flight large one: it + // must complete promptly rather than queue behind the semaphore it never + // touches. + smallResp := ts.claim(t, batchApplyPath, smallBody) + if smallResp.StatusCode != http.StatusOK { + t.Fatalf("small concurrent request status = %d, want 200: %s", smallResp.StatusCode, readAll(t, smallResp)) + } + + // Release the gate: both big calls (the first already inside ApplyBatch, + // the second still queued on the semaphore) can now finish in turn. + close(applier.gate) + + for i, ch := range []chan claimResult{firstDone, secondDone} { + select { + case res := <-ch: + if res.err != nil { + t.Errorf("large request %d transport error: %v", i, res.err) + continue + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("large request %d status = %d, want 200: %s", i, res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(5 * time.Second): + t.Fatalf("large request %d never completed after the gate opened", i) + } + } + + if got := applier.callCount(); got != 3 { + t.Fatalf("total ApplyBatch calls = %d, want 3 (two large, one small)", got) + } +} + +// waitForCallCount polls until applier has recorded at least n calls or the +// test's patience runs out. Used only to synchronize a goroutine with the +// point where a handler has entered the role, not to assert timing. +func waitForCallCount(t *testing.T, applier *controlledApplier, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if applier.callCount() >= n { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("ApplyBatch was not called %d time(s) within the wait window (got %d)", n, applier.callCount()) +} + +// claimResult carries an HTTP response or a transport error, so a goroutine +// can report either back to the test goroutine over a channel. Go testing +// disallows t.Fatal/t.Fatalf (and every other *testing.T failure method) from +// any goroutine but the one running the test function — no amount of internal +// serialization inside *testing.T changes that — so a helper meant to be +// called from a spawned goroutine must never call one, which is why this type +// exists and why claimAsync below reports through it instead. +type claimResult struct { + resp *http.Response + err error +} + +// statusOf reports a claimResult's status code, or -1 for a transport error, +// for a failure message that needs to read one without risking a nil +// dereference. +func statusOf(r claimResult) int { + if r.resp == nil { + return -1 + } + return r.resp.StatusCode +} + +// claimAsync is ts.claim's non-fatal twin: safe to call from any goroutine, +// because it never calls a *testing.T method. A caller reads the returned +// claimResult back on the TEST goroutine and fails there if it carries an +// error. +func (ts *testServer) claimAsync(body string) claimResult { + req, err := http.NewRequest(http.MethodPost, ts.base+batchApplyPath, strings.NewReader(body)) + if err != nil { + return claimResult{err: fmt.Errorf("new request: %w", err)} + } + req.Header.Set("Content-Type", "application/json") + resp, err := ts.client.Do(req) + if err != nil { + return claimResult{err: fmt.Errorf("POST %s: %w", batchApplyPath, err)} + } + return claimResult{resp: resp} +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 50b5ad53f..acc417999 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -39,6 +39,26 @@ const ProjectIDHeader = "Bd-Project-Id" // instead of discovering an older server silently ignored its stamp. const CapProjectEnforce = "project.enforce" +// CapBatchApplyLarge is the behavior capability that advertises the raised +// issues:batchApply envelope: up to issueops.MaxApplyBatchItems (1000) +// items, a maxApplyBatchBodyBytes (16 MiB) body, and — for a request over +// largeApplyItemThreshold items — an EXTENDED, operator-configurable run +// budget (Server.largeApplyCeiling, batch_apply.go / server.go) on top of +// the ordinary requestDeadline every request already gets; see +// Server.acquireLargeApply for how that budget is built and why it never +// narrows a small request's deadline. +// +// This token names all three limits together, and +// TestCapBatchApplyLargeTiesAllThreeLimits pins that: a build that lowers +// one of the three without removing this token would advertise an envelope +// it does not actually honor. Like CapProjectEnforce it names a server-wide +// BEHAVIOR rather than a route — issues.batchApply itself is already the +// per-operation token — so an older client that checks capabilities before +// sending an over-100-item plan can tell whether THIS server accepts it +// before it dials, rather than discovering a 400 only after paying for the +// round trip an old server would refuse anyway. +const CapBatchApplyLarge = "issues.batchApplyLarge" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -480,8 +500,18 @@ var routeTable = []route{ // ordered plan of four verbs whose items may reference each other — and a // flag on that operation would have made one operationId carrying two // contracts, two request schemas and two result shapes. - pattern: "/v0/beads/issues:batchApply", - capability: "issues.batchApply", + pattern: "/v0/beads/issues:batchApply", + capability: "issues.batchApply", + // This row carries NO maxDeadline: route() gives every request here + // the same unconditional requestDeadline (60s) every other route + // gets, so a request at or under largeApplyItemThreshold items runs + // under EXACTLY the deadline it always has. A request that crosses + // the threshold gets a whole separate, EXTENDED budget — + // s.largeApplyCeiling, an operator flag defaulting to 5 minutes — + // built fresh the moment it acquires the one-wide large-apply slot + // (Server.acquireLargeApply, server.go). See issues.batchApplyLarge + // (CapBatchApplyLarge) below, the token that advertises this + // envelope exists. implemented: true, handler: (*Server).handleApplyBatch, }, @@ -748,7 +778,7 @@ func (r route) specPathOf() string { // route. project.enforce announces per-request Bd-Project-Id enforcement // (checkProjectStamp): a stamped client reads it to know the refusal is available // rather than silently dropped by an older server. -var behaviorCapabilities = []string{CapProjectEnforce} +var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge} // Capabilities lists what this build advertises in ContextResponse.capabilities: // the operations it actually implements, gated on `implemented` so a stub can diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index b07fcd920..2c4e8def5 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -47,6 +47,20 @@ const ( // maxInflight bounds handlers that touch the database. Every unit of work // pins one SQL connection, so this is also the steady-state connection // count. + // + // It also bounds a request body's decode memory, worst case: every + // inflight slot is free to decode a full maxApplyBatchBodyBytes body + // concurrently before either the large-apply semaphore or anything else + // narrows it further, so the raw-bytes ceiling for POST issues:batchApply + // alone is maxInflight * maxApplyBatchBodyBytes (16 * 16 MiB = 256 MiB at + // today's defaults), and the decoded Go structures (strings, maps, slices + // of *CreateItem etc.) cost a further multiple of that on top — operators + // sizing this process's memory limit should budget for it. This is not + // actively capped narrower than maxInflight itself: only one of those + // concurrent decodes can go on to hold the one-wide large-apply slot + // (acquireLargeApply), so a decode that loses the race is refused with + // ErrBusy and its now-unreferenced ApplyBatchRequest is left for the next + // GC cycle — nothing in this file extends its lifetime past that refusal. maxInflight = 16 // maxConns bounds ACCEPTED connections. The semaphore does not: Go spawns // a goroutine per connection, and one parked on a full semaphore still @@ -70,8 +84,24 @@ const ( // the commit, so a graceful shutdown does not kill a connection whose write // may already have landed. drainTimeout = 20 * time.Second + // drainGrace is added on top of an in-flight large apply's own remaining + // deadline when extending the drain budget, so the response has time to + // be written back to the client after that apply's transaction commits + // right at its deadline, instead of the drain ending the instant the + // deadline is reached. + drainGrace = 5 * time.Second // uowCloseTimeout bounds the DETACHED close described on WithUOW. uowCloseTimeout = 5 * time.Second + // defaultLargeApplyCeiling is Config.LargeApplyCeiling's default: the + // whole-run budget a POST issues:batchApply request over + // largeApplyItemThreshold items gets once it holds the one-wide + // large-apply slot (acquireLargeApply). It is an operator-configurable + // CEILING rather than a per-request formula — every request that clears + // the threshold gets this whole budget from the moment it acquires the + // slot, whether it carries 101 items or 1000, because scaling it down for + // a request well under the envelope's top end would narrow a caller's + // budget for a reason unrelated to what it actually sent. + defaultLargeApplyCeiling = 5 * time.Minute ) // Pool limits for the provider's *sql.DB. The semaphore bounds handlers, not @@ -135,6 +165,26 @@ type Config struct { // service DNS name, which the rebinding defense would otherwise refuse; // see newHostPolicy. Empty leaves today's policy exactly as it was. AllowedHosts []string + // LargeApplyCeiling bounds how long POST /v0/beads/issues:batchApply may + // hold a write transaction for a request over largeApplyItemThreshold + // items (batch_apply.go). Zero takes defaultLargeApplyCeiling (5 + // minutes). It is wired to `bd serve --large-apply-ceiling`. + // + // It is a CEILING, not a target — see defaultLargeApplyCeiling for why + // every request over the threshold gets the whole budget rather than a + // value scaled to its item count. Raising or lowering it is an + // operational decision, not a wire change: it does not move + // largeApplyItemThreshold, maxApplyBatchBodyBytes or + // issueops.MaxApplyBatchItems, which is why TestCapBatchApplyLargeTiesAllThreeLimits + // exists — a build that lowers this without also removing the + // issues.batchApplyLarge capability token would otherwise advertise a + // budget it no longer honors. + // + // The orchestrator's stop grace must be at least this long: Serve's drain + // extends its budget to match an in-flight large apply's deadline, but a + // SIGKILL from outside this process does not consult it. See + // engdocs/SERVE_RUNBOOK.md. + LargeApplyCeiling time.Duration // Provider is where every database-touching handler opens its one unit of // work per request. Provider uow.UnitOfWorkProvider @@ -381,17 +431,67 @@ type Server struct { // sem bounds handlers that touch the database. Buffered channel rather // than sync.Semaphore so the acquisition can select on a timer. sem chan struct{} + // largeApplySem is the ONE-WIDE "large write" semaphore: a POST + // issues:batchApply request carrying more than largeApplyItemThreshold + // items holds this slot for the whole call, so at most one oversized + // batch-apply transaction runs at a time no matter how many ordinary + // requests sem admits concurrently. It is separate from sem (which + // bounds ALL database-touching handlers) because the two answer + // different questions: sem is "how many handlers may run", this is "how + // many may hold a write transaction open for minutes". A small apply + // never touches it. + // + // The wait to acquire this slot is itself BOUNDED — see + // acquireLargeApply — deliberately: a goroutine queued here still holds + // its sem slot (route() acquires sem before the handler ever runs), so + // an unbounded wait here would starve every other request behind a full + // sem the instant enough large applies queued up. + largeApplySem chan struct{} + // largeApplyCeiling is the resolved Config.LargeApplyCeiling (defaulted + // via orDefault to defaultLargeApplyCeiling). acquireLargeApply builds + // each run's deadline from this, fresh, at the moment the slot is + // acquired — never before, so time spent queued is never charged against + // the run itself. + largeApplyCeiling time.Duration + // largeApplyDeadline is the wall-clock deadline of the large apply + // currently holding largeApplySem, or nil when none is in flight. + // Serve's graceful-drain shutdown reads it to decide how long to wait + // for that request rather than cutting it off at the ordinary + // drainTimeout: a graceful drain waits for an in-flight large apply up to + // its own deadline, and — because acquireLargeApply refuses a NEW + // acquisition once s.closing fires — that in-flight request is the only + // one the budget ever has to cover. + largeApplyDeadline atomic.Pointer[time.Time] + // largeApplyWaiting gates the BOUNDED wait in acquireLargeApply to at + // most one goroutine at a time. Without this, every large request beyond + // the one holding largeApplySem would still queue for up to semTimeout + // waiting on the slot — and each queued goroutine holds its own sem slot + // the whole time it waits (route() acquires sem before the handler runs), + // so a burst of large requests could occupy up to semTimeout worth of + // general concurrency slots simply queueing, starving ordinary small + // requests for that whole window even though only one of them could ever + // win the large-apply slot. A second large request arriving while one is + // already queued is refused immediately (ErrBusy, no wait at all) instead + // of queueing behind it. + largeApplyWaiting atomic.Bool // auth is nil on an unauthenticated server, which is the loopback default. auth *TokenFileAuth - // semTimeout, semWarn, writeStall, watchPoll and watchBeat default to the - // constants above. They are fields rather than constants at the point of use - // so the queueing, stalled-write and streaming behavior can be exercised in - // milliseconds instead of tens of seconds. + // semTimeout, semWarn, writeStall, watchPoll, watchBeat and drainTimeout + // default to the constants above. They are fields rather than constants + // at the point of use so the queueing, stalled-write, streaming and + // drain behavior can be exercised in milliseconds instead of tens of + // seconds. semTimeout time.Duration semWarn time.Duration writeStall time.Duration watchPoll time.Duration watchBeat time.Duration + // drainTimeout overrides the package-level drainTimeout constant used as + // drainBudget's floor. A test shrinks this to prove Serve's drain + // actually EXTENDS past it for an in-flight large apply, instead of + // merely asserting the pure drainBudget helper's return value — see + // TestServeExtendsDrainPastAFixedFloorForAnInFlightLargeApply. + drainTimeout time.Duration // closing is closed when a graceful shutdown begins, and it is the ONLY // notice a streaming handler gets: http.Server.Shutdown waits for active @@ -527,9 +627,11 @@ func Listen(cfg Config) (*Server, error) { workspaceMemories: cfg.Memories, eventsJournal: cfg.EventsJournal, - sem: make(chan struct{}, maxInflight), - semTimeout: semAcquireTimeout, - semWarn: saturationWarn, + sem: make(chan struct{}, maxInflight), + semTimeout: semAcquireTimeout, + semWarn: saturationWarn, + largeApplySem: make(chan struct{}, 1), + largeApplyCeiling: orDefault(cfg.LargeApplyCeiling, defaultLargeApplyCeiling), closing: make(chan struct{}), maxWatchStreams: maxWatchStreams, @@ -683,6 +785,31 @@ func checkDatabaseSource(cfg Config) error { // the ephemeral default. func (s *Server) Addr() string { return s.listener.Addr().String() } +// drainBudget computes how long Serve should wait for in-flight work to +// finish once shutdown begins. It is a pure function, pulled out of Serve so +// the EXTENSION behavior described on Serve's doc comment — a large apply +// still in flight gets AT LEAST the remainder of its own deadline, not just +// the ordinary drainTimeout — can be pinned directly by a fast unit test +// (TestDrainBudget) instead of only indirectly, via a real multi-second wait. +// +// base is the floor Serve uses when no large apply is in flight — ordinarily +// the package-level drainTimeout constant, but overridable per-Server (see +// Server.drainTimeout) so a test can shrink it and observe Serve's ACTUAL +// drain genuinely outlasting that floor for an in-flight large apply, rather +// than only asserting this helper's return value in isolation. +// +// largeApplyDeadline is s.largeApplyDeadline.Load()'s result: nil when no +// large apply holds the slot at the moment shutdown starts. +func drainBudget(base time.Duration, largeApplyDeadline *time.Time) time.Duration { + budget := base + if largeApplyDeadline != nil { + if remaining := time.Until(*largeApplyDeadline); remaining > budget { + budget = remaining + drainGrace + } + } + return budget +} + // Serve accepts requests until ctx is canceled, then drains. It returns nil // on a clean shutdown; a listener failure is returned as-is. // @@ -690,6 +817,32 @@ func (s *Server) Addr() string { return s.listener.Addr().String() } // Shutdown does not cancel in-flight handler contexts: killing such a // connection early would leave the client unable to tell whether its write // landed. +// +// A large batch-apply can legitimately still be running at up to its own +// largeApplyCeiling (an operator flag, 5 minutes by default) when shutdown +// begins — far past the ordinary drainTimeout — so the budget here EXTENDS to +// that request's own deadline when one is in flight (s.largeApplyDeadline), +// instead of killing a transaction that may already be minutes into +// committing. An ordinary shutdown with no large apply in flight is +// unaffected: it still drains in drainTimeout. +// +// s.closing is closed here, BEFORE the budget is computed, rather than left +// to fire only when http.Server.Shutdown is called below: RegisterOnShutdown +// hooks (closeStreams among them) run when Shutdown is CALLED, not before, so +// computing the budget first would leave a window where a large apply could +// still start — and admitLargeApply's own post-acquisition recheck of +// s.closing is the second gate that closes that window from the other side. +// +// This is safe to compute ONCE, at shutdown start, because +// acquireLargeApply refuses every NEW acquisition of the large-apply slot +// once s.closing fires: the in-flight request this budget was sized for is +// the only one that can still be holding the slot by now, never a later +// arrival that queued after the snapshot. +// +// The operator's orchestrator stop grace must be AT LEAST largeApplyCeiling +// plus drainGrace, or a SIGKILL from outside this process can still cut off +// an in-flight large apply this drain would otherwise have waited out; see +// engdocs/SERVE_RUNBOOK.md. func (s *Server) Serve(ctx context.Context) error { errCh := make(chan error, 1) go func() { errCh <- s.http.Serve(s.listener) }() @@ -703,10 +856,17 @@ func (s *Server) Serve(ctx context.Context) error { case <-ctx.Done(): } - s.event("shutdown_start", "drain_timeout", drainTimeout.String(), "conns", s.liveConns.Load()) + // Close s.closing before computing the drain budget (see doc comment + // above): closeStreams is idempotent (sync.Once), so it is safe to call + // here AND still leave it registered via RegisterOnShutdown for when + // Shutdown runs below. + s.closeStreams() + + budget := drainBudget(orDefault(s.drainTimeout, drainTimeout), s.largeApplyDeadline.Load()) + s.event("shutdown_start", "drain_timeout", budget.String(), "conns", s.liveConns.Load()) // Detached: ctx is already canceled, and the drain is the point. - drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), drainTimeout) + drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget) defer cancel() if err := s.http.Shutdown(drainCtx); err != nil { @@ -1301,6 +1461,121 @@ func orDefault(v, fallback time.Duration) time.Duration { return fallback } +// acquireLargeApply takes the one-wide large-write slot for a POST +// issues:batchApply request over largeApplyItemThreshold items, and builds +// that request's run context. +// +// THE WAIT IS BOUNDED, on the SAME orDefault(s.semTimeout, semAcquireTimeout) +// budget acquire uses for the ordinary sem — not the caller's own request +// deadline, and not unbounded. A goroutine queued here still holds its +// general sem slot (route() acquires sem before the handler runs, and holds +// it for the handler's whole lifetime), so an unbounded wait would let enough +// queued large applies starve every other request behind a saturated sem; +// see TestLargeApplyQueueDoesNotStarveOrdinaryRequests. Every refusal this +// function can produce — the bounded wait expiring, s.closing firing because +// a graceful shutdown began, or the caller's own ctx.Done() firing while +// queued — answers ErrBusy, never a raw ctx.Err(): nothing was written in any +// of those cases, so the caller gets a 503 with Retry-After rather than the +// non-retryable 500 a bare context error would classify as (ClassifyError +// deliberately excludes context errors from its 503 path for OTHER callers, +// where a context error can mean a write is in flight — that exclusion does +// not apply here, because this slot guards nothing but queue position). +// +// THE RUN DEADLINE IS STAMPED ONLY ON SUCCESS, and only once the slot is +// actually held: runCtx is a DETACHED copy of ctx (context.WithoutCancel) +// given a fresh largeApplyCeiling-wide timeout starting NOW, not narrowed +// from whatever deadline ctx already carried and not started back when the +// request first queued. That is what makes this an EXTENSION of the small +// request's ordinary budget rather than a narrowing of it: every request +// under largeApplyItemThreshold items runs under route()'s own unconditional +// requestDeadline and never reaches this function at all, and a request that +// clears the threshold gets a whole fresh largeApplyCeiling on top of +// whatever it already spent queued. Detaching from ctx's cancellation means a +// client disconnect mid-run does not abort an already-admitted large apply — +// deliberate, because it is a single atomic write that should not be +// abandoned half-considered merely because the caller stopped reading; the +// same trade-off WithUOW's own deferred close already makes for a request +// context. +// +// release cancels runCtx, clears s.largeApplyDeadline and frees the slot, in +// that order, so nothing observes the deadline as cleared while the run +// context is still live. +func (s *Server) acquireLargeApply(ctx context.Context) (runCtx context.Context, release func(), err error) { + select { + case <-s.closing: + return nil, nil, ErrBusy + default: + } + select { + case s.largeApplySem <- struct{}{}: + return s.admitLargeApply(ctx) + default: + } + + // At most ONE caller may queue for the slot at a time. Without this gate, + // every large request beyond the one already holding largeApplySem would + // still wait up to semTimeout below — and each of those waiters holds its + // own sem slot the whole time (route() acquires sem before the handler + // runs) — so a burst of large requests could occupy a semTimeout's worth + // of general concurrency slots merely queueing, starving small requests + // for that whole window even though only one waiter could ever win the + // large-apply slot. A second large request arriving while one is already + // queued is refused immediately: no wait, no sem slot held for it. + if !s.largeApplyWaiting.CompareAndSwap(false, true) { + return nil, nil, ErrBusy + } + defer s.largeApplyWaiting.Store(false) + + timer := time.NewTimer(orDefault(s.semTimeout, semAcquireTimeout)) + defer timer.Stop() + select { + case s.largeApplySem <- struct{}{}: + return s.admitLargeApply(ctx) + case <-timer.C: + return nil, nil, ErrBusy + case <-s.closing: + return nil, nil, ErrBusy + case <-ctx.Done(): + // The client hung up, or the request's own (unrelated, ordinary) + // deadline expired, while still queued for the slot. Nothing was + // written — this path never opened a transaction — so it is the same + // ErrBusy every other refusal above answers, not ctx.Err(). + return nil, nil, ErrBusy + } +} + +// admitLargeApply runs once acquireLargeApply has actually taken the slot: it +// builds the run's own detached, freshly-timed context and stamps +// s.largeApplyDeadline for Serve's graceful-drain budget to read. +// +// It re-checks s.closing AFTER taking the slot and storing the deadline, +// because a select with multiple ready cases (largeApplySem and s.closing +// both ready at once, in acquireLargeApply above) picks pseudo-randomly: the +// slot can be won in the same instant a graceful shutdown begins, racing +// ahead of Serve's drainBudget snapshot. Losing that race here — releasing +// the slot and refusing with ErrBusy — is what keeps Serve's "the in-flight +// request this budget was sized for is the only one that can still be +// holding the slot" invariant true. +func (s *Server) admitLargeApply(ctx context.Context) (runCtx context.Context, release func(), err error) { + runCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), orDefault(s.largeApplyCeiling, defaultLargeApplyCeiling)) + deadline, _ := runCtx.Deadline() + s.largeApplyDeadline.Store(&deadline) + release = func() { + s.largeApplyDeadline.Store(nil) + cancel() + <-s.largeApplySem + } + + select { + case <-s.closing: + release() + return nil, nil, ErrBusy + default: + } + + return runCtx, release, nil +} + // handler builds the whole request path: the route table's registrations, the // catch-all that keeps unrouted paths on the same error shape, and the // middleware in front of both. @@ -1956,6 +2231,12 @@ func (s *Server) logStartup() { "max_conns", maxConns, "sem_wait", semAcquireTimeout.String(), "deadline", requestDeadline.String(), + // large_apply_ceiling is the PER-ROUTE budget a POST + // issues:batchApply request over largeApplyItemThreshold items gets, + // separate from the ordinary "deadline" above: an operator reading + // this line must not conclude every request here is bounded by the + // 60s "deadline" figure just because that is the only one printed. + "large_apply_ceiling", orDefault(s.largeApplyCeiling, defaultLargeApplyCeiling).String(), } // The pool bounds are this server's, applied to the provider above. On the // roles source there is no pool here to bound, and printing the numbers diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index f84f419fb..6bae6ef99 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -632,10 +632,10 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { t.Fatal("the route table contributed no capabilities; this case would pass against a server that advertises nothing") } // The behavior tokens name no route, so the table walk above cannot reach - // them; project.enforce is advertised in the same list. It is spelled - // literally here to keep this an independent oracle rather than a second - // call to the code under test. - want = append(want, "project.enforce") + // them; project.enforce and issues.batchApplyLarge are advertised in the + // same list. Both are spelled literally here to keep this an independent + // oracle rather than a second call to the code under test. + want = append(want, "project.enforce", "issues.batchApplyLarge") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 4fde46144..38bc0c0a2 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -7449,12 +7449,26 @@ components: `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, - `issues.casMetadata`; the one behavior token is `project.enforce`, + `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace - is refused here rather than silently ignored. The list grows - additively, and an operation never appears here unless it is fully - implemented. This is how a client checks for an operation or a - behavior — never the version string. + is refused here rather than silently ignored, and + `issues.batchApplyLarge`, which announces that + `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — + up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB + body, and (for a request over 100 items) a run budget that + EXTENDS to an operator-configured ceiling (`bd serve + --large-apply-ceiling`, 5 minutes by default) — rather than the + original 100-item, 4 MiB bound. The ceiling is a flat whole-run + budget, not a value scaled by item count: a 101-item request and a + 1000-item request both get the same extended budget. A request at + or under 100 items is unaffected either way and keeps the + server's ordinary deadline exactly as before. A client that + checks for this token before sending a plan over 100 items learns + whether THIS server accepts it without paying for a round trip an + older server would refuse anyway. The list grows additively, and + an operation never appears here unless it is fully implemented. + This is how a client checks for an operation or a behavior — + never the version string. THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which @@ -8796,7 +8810,7 @@ components: items: type: array minItems: 1 - maxItems: 100 + maxItems: 1000 description: >- The items to apply, IN THE ORDER THEY ARE TO BE APPLIED. An empty array is a `400` rather than a successful no-op: a write request that @@ -8804,10 +8818,20 @@ components: client whose own plan filtered to nothing silently stops writing. - The 100-item cap bounds how long one request may hold a write - transaction, not batch semantics. Split a larger plan; each request - is atomic on its own — but splitting it changes what the end gate - can see, since the gate runs over one request at a time. + The 1000-item cap (raised from 100; see the `issues.batchApplyLarge` + capability token on `ContextResponse`) bounds how long one request + may hold a write transaction, not batch semantics. A request over + 100 items also EXTENDS the effective request deadline: the server + grants a flat, operator-configured ceiling (`bd serve + --large-apply-ceiling`, 5 minutes by default) instead of the + ordinary small-request deadline, so a plan that genuinely needs + minutes to apply is not cut off mid-commit. It never narrows: a + request at or under 100 items keeps exactly the deadline it has + always had. Split a larger plan if you must; each request is + atomic on its own — but splitting it changes what the end gate + can see, since the gate runs over one request at a time, and the + server itself never splits a plan you send it: a single `items` + array either commits in full or changes nothing. A per-item refusal names its offender as `items[i].kind.member`. diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index eb25b139e..ec88277a8 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -136,6 +136,7 @@ filegroup( ":go_srcs", "//internal/storage/backendnames:go_srcs", "//internal/storage/backends:go_srcs", + "//internal/storage/batchfixtures:go_srcs", "//internal/storage/contextinfo:go_srcs", "//internal/storage/dberrors:go_srcs", "//internal/storage/dbproxy/identity:go_srcs", @@ -163,6 +164,7 @@ filegroup( "//internal/storage/schema:go_srcs", "//internal/storage/sqlbuild:go_srcs", "//internal/storage/sqlclass:go_srcs", + "//internal/storage/sqlcount:go_srcs", "//internal/storage/storagecontract:go_srcs", "//internal/storage/uow:go_srcs", "//internal/storage/versioncontrolops:go_srcs", diff --git a/internal/storage/batchfixtures/BUILD.bazel b/internal/storage/batchfixtures/BUILD.bazel new file mode 100644 index 000000000..9afbdd6a5 --- /dev/null +++ b/internal/storage/batchfixtures/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "batchfixtures", + srcs = [ + "batchfixtures.go", + "uncapped_plan.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/batchfixtures", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//issueops", + ], +) + +go_test( + name = "batchfixtures_test", + srcs = ["batchfixtures_test.go"], + deps = [ + ":batchfixtures", + "//internal/storage", + "//issueops", + ], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/batchfixtures/batchfixtures.go b/internal/storage/batchfixtures/batchfixtures.go index fb286e987..cb1ad6958 100644 --- a/internal/storage/batchfixtures/batchfixtures.go +++ b/internal/storage/batchfixtures/batchfixtures.go @@ -1,5 +1,5 @@ // Package batchfixtures builds issueops.ApplyBatchRequest values shaped like -// the plans S5b-3-LARGE-BATCH-DESIGN.md measures: the mol-normalized +// the plans measured for the large-batch-apply work: the mol-normalized // issue/PR work plan (and its 2x variant) and a "classic" hand-authored // plan. It does no I/O — every fixture here is pure data construction — so a // benchmark or test in any backend package can import it without risking an @@ -23,8 +23,7 @@ func createKey(prefix string, n int) string { // molCounts is the sub-item composition mol-normalized-issue-pr-work-v1 // measures at 1x: 102 creates, 136 blocks edges, 102 tracks-to-root edges, -// 16 assign updates == 356 items total. See S5b-3-LARGE-BATCH-DESIGN.md -// "Measured sizes". +// 16 assign updates == 356 items total. type molCounts struct { creates int blocksEdges int @@ -42,6 +41,13 @@ var mol1x = molCounts{creates: 102, blocksEdges: 136, tracksEdges: 102, assignIt // makes them comparable points on the same curve. var mol2x = molCounts{creates: 204, blocksEdges: 272, tracksEdges: 204, assignItems: 32} +// molCap is mol1x's sub-category ratio scaled up to land exactly on +// issueops.MaxApplyBatchItems (1000): the top of the large-apply envelope the +// server's --large-apply-ceiling default (5 minutes) is sized against, so a +// wall-clock measurement at this shape is a measurement AT the cap, not an +// extrapolation from a smaller one. +var molCap = molCounts{creates: 286, blocksEdges: 382, tracksEdges: 286, assignItems: 46} + // buildMolShape builds a mol-normalized-issue-pr-work-v1-shaped request: a // flat batch of work-item creates (each carrying metadata), a blocks chain // wiring them together, every created item tracking back to an @@ -151,6 +157,15 @@ func Shape712(actor, rootID string) issueops.ApplyBatchRequest { return buildMolShape(actor, "mol2x", mol2x, rootID) } +// Shape1000 builds a plan at exactly issueops.MaxApplyBatchItems (286 +// creates, 668 edges [382 blocks + 286 tracks], 46 assign updates), for 1000 +// items total: the top of the large-apply envelope, for measuring wall-clock +// time at the cap rather than only at the design doc's 356/712 sample +// points. rootID must already exist in the target store. +func Shape1000(actor, rootID string) issueops.ApplyBatchRequest { + return buildMolShape(actor, "molcap", molCap, rootID) +} + // ShapeClassic40 builds a 40-item "classic" (non-mol) plan: 10 creates, 20 // edges (12 blocks + 8 parent-child) and 10 assign updates. // diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index f39bc8370..8183faa64 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -260,6 +260,7 @@ go_test( "keyset_search_test.go", "label_rename_test.go", "labels_test.go", + "large_batch_apply_measure_test.go", "lease_migration_test.go", "lease_test.go", "lenient_open_dirty_tables_integration_test.go", @@ -384,6 +385,7 @@ go_test( "//internal/doltserver", "//internal/githooksenv", "//internal/storage", + "//internal/storage/batchfixtures", "//internal/storage/dberrors", "//internal/storage/depid", "//internal/storage/doltutil", @@ -393,6 +395,7 @@ go_test( "//internal/storage/rowid", "//internal/storage/schema", "//internal/storage/sqlbuild", + "//internal/storage/sqlcount", "//internal/storage/storagecontract", "//internal/storage/versioncontrolops", "//internal/testutil", diff --git a/internal/storage/dolt/large_batch_apply_measure_test.go b/internal/storage/dolt/large_batch_apply_measure_test.go index db60b6a9f..e8714ba28 100644 --- a/internal/storage/dolt/large_batch_apply_measure_test.go +++ b/internal/storage/dolt/large_batch_apply_measure_test.go @@ -1,8 +1,7 @@ package dolt -// B0 (S5b-3-LARGE-BATCH-DESIGN.md): measure the ACTUAL number of SQL -// statements issueops.ApplyBatchInTx issues on the Dolt server (TCP) -// backend, for the design's three measured plan shapes. See +// Measure the ACTUAL number of SQL statements issueops.ApplyBatchInTx issues +// on the Dolt server (TCP) backend, for three measured plan shapes. See // internal/storage/embeddeddolt/large_batch_apply_measure_test.go for the // embedded backend equivalent, and internal/storage/batchfixtures for the // shared, backend-agnostic plan construction. @@ -109,8 +108,7 @@ var largeBatchApplyDoltShapes = []struct { // TestLargeBatchApplyStatementCounts_Dolt pins the ACTUAL number of SQL // statements issueops.ApplyBatchInTx issues on the Dolt server (TCP) -// backend, for each of the design's three measured shapes -// (S5b-3-LARGE-BATCH-DESIGN.md, slice B0). Sibling regression baseline to +// backend, for each of three measured shapes. Sibling regression baseline to // TestLargeBatchApplyStatementCounts_Embedded in the embeddeddolt package — // see that test's doc comment for why counts are pinned with a small // tolerance rather than exactly, and why a drift beyond it should be diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 1adcaccf5..8d6f09dfc 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -158,6 +158,7 @@ go_test( "journal_truncation_test.go", "label_list_all_benchmark_test.go", "label_rename_test.go", + "large_batch_apply_measure_test.go", "lease_replica_test.go", "lease_test.go", "lenient_gate_warning_test.go", @@ -228,6 +229,7 @@ go_test( "//internal/eventsjournal", "//internal/hooks", "//internal/storage", + "//internal/storage/batchfixtures", "//internal/storage/dberrors", "//internal/storage/depid", "//internal/storage/doltutil", @@ -235,15 +237,18 @@ go_test( "//internal/storage/externaldeps", "//internal/storage/issueops", "//internal/storage/schema", + "//internal/storage/sqlcount", "//internal/storage/storagecontract", "//internal/storage/versioncontrolops", "//internal/testutil/bazeltest", "//internal/types", "//internal/workapi", "//issueops", + "@com_github_cenkalti_backoff_v4//:backoff", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_dolthub_dolt_go//libraries/doltcore/dbfactory", "@com_github_dolthub_dolt_go//store/blobstore", + "@com_github_dolthub_driver_v2//:driver", "@com_github_google_uuid//:uuid", "@org_golang_x_sync//errgroup", ], diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go index 200174b58..59f0e5574 100644 --- a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -2,9 +2,8 @@ package embeddeddolt_test -// B0 (S5b-3-LARGE-BATCH-DESIGN.md): measure the ACTUAL number of SQL -// statements issueops.ApplyBatchInTx issues on the embedded Dolt backend, -// for the design's three measured plan shapes. See +// B0: measure the ACTUAL number of SQL statements issueops.ApplyBatchInTx +// issues on the embedded Dolt backend, for three measured plan shapes. See // internal/storage/dolt/large_batch_apply_measure_test.go for the Dolt // server (TCP) backend equivalent, and internal/storage/batchfixtures for // the shared, backend-agnostic plan construction. @@ -169,8 +168,8 @@ var largeBatchApplyShapes = []struct { // TestLargeBatchApplyStatementCounts_Embedded pins the ACTUAL number of SQL // statements issueops.ApplyBatchInTx issues on the embedded backend, for -// each of the design's three measured shapes (S5b-3-LARGE-BATCH-DESIGN.md, -// slice B0). It is a REGRESSION BASELINE for B2 (a later, lighter fast +// each of three measured shapes (slice B0). It is a REGRESSION BASELINE for +// B2 (a later, lighter fast // path): B2 must lower these numbers, and this test is what proves it did. // // The exact counts are backend- and Dolt-version-sensitive by nature — they @@ -227,6 +226,76 @@ func TestLargeBatchApplyStatementCounts_Embedded(t *testing.T) { } } +// wallClockShapes is the coordinator's item-8 ask: real wall-clock numbers at +// 356 items (the design doc's primary measured shape) and 1000 items (the +// hard cap, issueops.MaxApplyBatchItems) — not just statement counts — so the +// server's --large-apply-ceiling default (5 minutes) is backed by an actual +// measurement at the top of the envelope, not only extrapolated from smaller +// shapes. +var wallClockShapes = []struct { + name string + db string + build func(rootID string) issueops.ApplyBatchRequest +}{ + {name: "356", db: "wc356", build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape356("tester", rootID) + }}, + {name: "1000", db: "wc1000", build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape1000("tester", rootID) + }}, +} + +// TestLargeBatchApplyWallClock_Embedded measures (and simply logs, rather +// than asserting a bound on) how long issueops.ApplyBatchInTx actually takes +// on the embedded backend at 356 and 1000 items. It is deliberately NOT a +// pass/fail regression gate — wall-clock time is host- and CI-runner- +// sensitive in a way statement counts are not — but the numbers it logs are +// the real-data justification for defaultLargeApplyCeiling +// (internal/httpapi/server.go, 5 minutes): both measured shapes must +// complete in a small fraction of that budget for the ceiling to be +// generous headroom rather than a number picked out of thin air. +func TestLargeBatchApplyWallClock_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + + for _, tc := range wallClockShapes { + t.Run(tc.name, func(t *testing.T) { + fixture := newPristineEmbeddedDoltFixture(t, tc.db) + t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := fixture.store.CreateIssue(ctx, root, "tester"); err != nil { + t.Fatalf("create root issue: %v", err) + } + + plan := tc.build(root.ID) + db, cleanup, err := openCountedConn(ctx, fixture.dataDir, fixture.database, &sqlcount.Counts{}) + if err != nil { + t.Fatalf("openCountedConn: %v", err) + } + defer cleanup() + + uncapped := batchfixtures.UncappedPlan(plan) + start := time.Now() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + if _, _, err := storageissueops.ApplyBatchInTx(ctx, tx, uncapped); err != nil { + _ = tx.Rollback() + t.Fatalf("ApplyBatchInTx: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + elapsed := time.Since(start) + + t.Logf("embedded wall-clock: %s items (%d actual) = %s (defaultLargeApplyCeiling headroom: %.1fx)", + tc.name, len(plan.Items), elapsed, (5*time.Minute).Seconds()/elapsed.Seconds()) + }) + } +} + // pinnedEmbeddedStatementCounts is the B0 regression baseline for the // embedded backend: the Total() statement count issueops.ApplyBatchInTx // issued for each measured shape, as last observed on this branch. B2 must diff --git a/internal/storage/sqlcount/BUILD.bazel b/internal/storage/sqlcount/BUILD.bazel new file mode 100644 index 000000000..66d03210a --- /dev/null +++ b/internal/storage/sqlcount/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "sqlcount", + srcs = ["sqlcount.go"], + importpath = "github.com/steveyegge/beads/internal/storage/sqlcount", + visibility = ["//:__subpackages__"], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/sqlcount/sqlcount.go b/internal/storage/sqlcount/sqlcount.go index 75d5f1516..958a6b76a 100644 --- a/internal/storage/sqlcount/sqlcount.go +++ b/internal/storage/sqlcount/sqlcount.go @@ -3,10 +3,9 @@ // against a real driver, and optionally inject artificial per-statement // latency to approximate a database that is not co-located with the caller. // -// It exists for the large-batch-apply measurement work -// (S5b-3-LARGE-BATCH-DESIGN.md, slice B0): issueops.ApplyBatchInTx takes a -// concrete *sql.Tx, not an interface, so the only seam available to count its -// real round trips is the database/sql/driver boundary itself. Wrapping the +// It exists for the large-batch-apply measurement work: issueops.ApplyBatchInTx +// takes a concrete *sql.Tx, not an interface, so the only seam available to +// count its real round trips is the database/sql/driver boundary itself. Wrapping the // driver.Connector a backend already builds — mysql.NewConnector for the Dolt // server backend, doltembed.NewConnector for the embedded backend — is the // one construction point both backends share, so one wrapper measures both diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 1a98ccda7..276d3f2e5 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -80,6 +80,7 @@ go_library( go_test( name = "uow_test", srcs = [ + "batch_applier_retry_budget_test.go", "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 1c8d0033f..aba391947 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "sort" + "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" @@ -40,6 +41,69 @@ func NewBatchApplier(provider UnitOfWorkProvider) (publicops.BatchApplier, error var _ publicops.BatchApplier = (*batchApplier)(nil) +// retryTxAttemptHeadroom is reserved off the top of an extended retry budget +// so the FINAL retry attempt still has room to run to completion before the +// caller's own context deadline arrives. Without it, an extended budget +// computed as exactly "whatever's left on ctx" could let backoff start one +// more attempt a moment before ctx expires: that attempt is then killed +// mid-transaction by ctx cancellation, turning a write that may have been +// about to succeed into a context-deadline-exceeded error (classified +// non-retryable, a 500) instead of a clean retry-budget exhaustion (a +// reported, retryable write conflict). The retry loop's OWN loser case +// already reports correctly (uow.RunTxResultWithin's IsSerializationError +// contract) — this headroom exists only to make ctx cancellation not race it. +const retryTxAttemptHeadroom = 5 * time.Second + +// retryTxMinBudget is the floor retryTxBudget returns for a MARKED context +// whose remaining deadline (after subtracting retryTxAttemptHeadroom) is too +// small to be a useful retry budget on its own — e.g. an operator-configured +// --large-apply-ceiling set close to expiry, or a request nearly at its own +// deadline. It deliberately does NOT fall back to DefaultTxRetryMaxElapsed +// (15s): claiming a 15s retry budget when ctx itself has only a second or two +// left would be misleading busywork, since ctx cancellation via +// backoff.WithContext ends the loop at its own deadline regardless of what +// MaxElapsedTime says. A small positive floor just guarantees backoff.Retry's +// unconditional first attempt (and typically one quick retry) rather than a +// degenerate zero-or-negative MaxElapsedTime, which the backoff library +// treats as "never stop on elapsed time" — the opposite of what a +// nearly-expired budget should mean. +const retryTxMinBudget = time.Second + +// retryTxBudget picks ApplyBatch's commit-retry budget. It returns exactly +// DefaultTxRetryMaxElapsed unless ctx was marked by +// issueops.WithExtendedRetryBudget, which only internal/httpapi's +// admitLargeApply does, for a request over the large-apply item threshold. +// +// A version of this fix that instead scaled to "any sufficiently long +// ctx.Deadline()" was wrong: an ORDINARY request also carries a real +// deadline (internal/httpapi's requestDeadline, 60s), and that heuristic +// silently raised every ordinary request's retry ceiling from 15s to up to +// 60s. Gating on the explicit marker keeps ordinary requests at EXACTLY the +// pre-existing 15s ceiling, and extends only the path that intentionally +// asked for more room: an over-threshold request gets a context deadline of +// up to its own largeApplyCeiling (5 minutes by default) precisely so a +// genuinely large, slow-to-commit write has room to run — but a Dolt +// commit-time merge conflict on it would otherwise still give up retrying at +// 15s, with minutes of that budget sitting unused. +// +// retryTxAttemptHeadroom is subtracted from the remaining deadline so the +// scaled budget never lets the last attempt start too late to finish; ctx +// cancellation via backoff.WithContext is still the final backstop either +// way, so this can never let a request run past the deadline it already had. +func retryTxBudget(ctx context.Context) time.Duration { + if !publicops.HasExtendedRetryBudget(ctx) { + return DefaultTxRetryMaxElapsed + } + deadline, ok := ctx.Deadline() + if !ok { + return DefaultTxRetryMaxElapsed + } + if budget := time.Until(deadline) - retryTxAttemptHeadroom; budget > retryTxMinBudget { + return budget + } + return retryTxMinBudget +} + // ApplyBatch applies every item in ONE unit of work and commits them together. // // THIS LEG IS A GENUINELY INDEPENDENT BODY, unlike the third leg of MetadataCAS @@ -68,7 +132,7 @@ func (o *batchApplier) ApplyBatch(ctx context.Context, request publicops.ApplyBa if err != nil { return publicops.ApplyBatchResult{}, err } - return RunTxResult(ctx, o.provider, func(ctx context.Context, uw UnitOfWork) (publicops.ApplyBatchResult, string, error) { + return RunTxResultWithin(ctx, o.provider, retryTxBudget(ctx), func(ctx context.Context, uw UnitOfWork) (publicops.ApplyBatchResult, string, error) { run := &uowApplyRun{ plan: plan, uw: uw, diff --git a/internal/storage/uow/batch_applier_retry_budget_test.go b/internal/storage/uow/batch_applier_retry_budget_test.go new file mode 100644 index 000000000..6df44c1a1 --- /dev/null +++ b/internal/storage/uow/batch_applier_retry_budget_test.go @@ -0,0 +1,192 @@ +package uow + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestRetryTxBudgetScalesToRemainingDeadline pins the fix for a large apply's +// retry budget: RunTxResult's hardcoded 15s ceiling must not silently disable +// retries for the bulk of a large apply's own (much longer) context deadline +// — but ONLY on the path that explicitly asks for that (marked via +// publicops.WithExtendedRetryBudget). An ordinary request's context also +// carries a real deadline (60s); an earlier version of this fix scaled the +// budget to ANY sufficiently long remaining deadline and silently raised +// ordinary requests from 15s to up to 60s of retrying. That regression is +// exactly what the unmarked subtests below pin against. +func TestRetryTxBudgetScalesToRemainingDeadline(t *testing.T) { + t.Run("no deadline, no marker gets the default", func(t *testing.T) { + got := retryTxBudget(context.Background()) + if got != DefaultTxRetryMaxElapsed { + t.Errorf("retryTxBudget(no deadline) = %s, want %s", got, DefaultTxRetryMaxElapsed) + } + }) + + t.Run("a deadline shorter than the default never shrinks the budget when unmarked", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + got := retryTxBudget(ctx) + if got != DefaultTxRetryMaxElapsed { + t.Errorf("retryTxBudget(1s deadline) = %s, want %s (never below the default when unmarked)", got, DefaultTxRetryMaxElapsed) + } + }) + + t.Run("an UNMARKED long deadline (an ordinary request's own 60s) stays at exactly the default", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + got := retryTxBudget(ctx) + if got != DefaultTxRetryMaxElapsed { + t.Errorf("retryTxBudget(60s deadline, unmarked) = %s, want exactly %s (an ordinary request must not retry longer just because its ctx happens to have a deadline)", got, DefaultTxRetryMaxElapsed) + } + }) + + t.Run("a MARKED deadline far past the default extends the budget to match it, minus attempt headroom", func(t *testing.T) { + const ceiling = 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), ceiling) + defer cancel() + ctx = publicops.WithExtendedRetryBudget(ctx) + got := retryTxBudget(ctx) + want := ceiling - retryTxAttemptHeadroom + const slack = 2 * time.Second + if got < want-slack || got > want { + t.Errorf("retryTxBudget(%s deadline, marked) = %s, want ~%s (extended to match a large apply's own run budget, minus headroom for the final attempt)", ceiling, got, want) + } + }) + + t.Run("a MARKED deadline too close to expiry to leave attempt headroom floors at retryTxMinBudget, not the 15s default", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + ctx = publicops.WithExtendedRetryBudget(ctx) + got := retryTxBudget(ctx) + if got != retryTxMinBudget { + t.Errorf("retryTxBudget(3s deadline, marked) = %s, want %s (too little headroom to extend; claiming the full 15s default here would be misleading since ctx cancellation ends the loop first regardless)", got, retryTxMinBudget) + } + }) +} + +// alwaysSerializationErrorProvider fails EVERY NewUOW call with a retryable +// serialization error, so a caller through RunTxResultWithin never reaches +// the work function at all. That is what makes it cheap: proving ApplyBatch +// itself (not just the retryTxBudget helper) actually wires retryTxBudget's +// return value into RunTxResultWithin needs no real domain use cases — +// issueops.BatchApplier's huge UnitOfWork surface is never touched, because +// NewUOW itself is the thing failing. +type alwaysSerializationErrorProvider struct { + calls atomic.Int32 +} + +func (p *alwaysSerializationErrorProvider) NewUOW(context.Context) (UnitOfWork, error) { + p.calls.Add(1) + return nil, newMySQLError(1213) // deadlock: uow.isSerializationError's retryable case +} + +func (p *alwaysSerializationErrorProvider) Close(context.Context) error { return nil } + +// minimalValidApplyBatchRequest is the smallest request storage.PlanApplyBatch +// accepts: one actor, one create item. It is never actually applied in the +// tests below — alwaysSerializationErrorProvider fails before uowApplyRun.apply +// ever runs — so its content only has to pass static plan validation. +func minimalValidApplyBatchRequest() publicops.ApplyBatchRequest { + return publicops.ApplyBatchRequest{ + Actor: "tester", + Items: []publicops.ApplyItem{ + { + Kind: publicops.ItemCreate, + Create: &publicops.CreateItem{ + Issue: &types.Issue{ + Title: "retry budget wiring probe", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }, + }, + }, + }, + } +} + +// TestApplyBatchHonorsAnExtendedRetryBudgetFromContext tests ApplyBatch +// ITSELF, not just the retryTxBudget helper — the gap the coordinator's +// review flagged: a mutation that hardcodes ApplyBatch's call to +// RunTxResultWithin(ctx, o.provider, DefaultTxRetryMaxElapsed, ...) instead of +// retryTxBudget(ctx) passed every existing test, because nothing exercised +// ApplyBatch with a marked context and actually observed which budget +// governed the retry loop. +// +// Both subtests share the SAME 2-second outer ctx deadline, so a mutation +// that ignores the marker makes them behave IDENTICALLY: +// +// - unmarked: retryTxBudget must return the full 15s default. Since 15s +// vastly exceeds the 2s ctx deadline, ctx's OWN cancellation ends the +// loop first, so the observed error is ctx's DeadlineExceeded, never a +// serialization error, and it takes close to the full 2s. +// - marked: retryTxBudget must return retryTxMinBudget (1s, per the +// "too close to expiry to extend" case above, since 2s of remaining +// deadline is less than retryTxAttemptHeadroom). That 1s budget is +// SHORTER than the 2s ctx deadline, so the retry loop exhausts on its +// own well under 2s, and the observed error is the serialization error +// itself. +// +// If ApplyBatch ignored retryTxBudget's return value and always used the 15s +// default, the marked case would look exactly like the unmarked one: it +// would run nearly the full 2s and end on ctx's DeadlineExceeded instead of +// exhausting fast on the serialization error, and this test would fail. +func TestApplyBatchHonorsAnExtendedRetryBudgetFromContext(t *testing.T) { + const outerDeadline = 2 * time.Second + + t.Run("unmarked: the 15s default outlasts the 2s ctx, so ctx itself ends the loop", func(t *testing.T) { + provider := &alwaysSerializationErrorProvider{} + applier, err := NewBatchApplier(provider) + if err != nil { + t.Fatalf("NewBatchApplier: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), outerDeadline) + defer cancel() + + start := time.Now() + _, err = applier.ApplyBatch(ctx, minimalValidApplyBatchRequest()) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected an error: NewUOW never succeeds") + } + if IsSerializationError(err) { + t.Errorf("err = %v is a serialization error; want ctx's own DeadlineExceeded, because the 15s default budget should still be governing and outlast this 2s ctx", err) + } + if elapsed < 1500*time.Millisecond { + t.Errorf("elapsed = %s, want close to %s: the loop ended too early for the 15s default to have been the governing budget", elapsed, outerDeadline) + } + if provider.calls.Load() < 2 { + t.Errorf("NewUOW calls = %d, want several retries before ctx ended the loop", provider.calls.Load()) + } + }) + + t.Run("marked: the too-short-to-extend 1s floor governs and exhausts well under the 2s ctx", func(t *testing.T) { + provider := &alwaysSerializationErrorProvider{} + applier, err := NewBatchApplier(provider) + if err != nil { + t.Fatalf("NewBatchApplier: %v", err) + } + ctx, cancel := context.WithTimeout(publicops.WithExtendedRetryBudget(context.Background()), outerDeadline) + defer cancel() + + start := time.Now() + _, err = applier.ApplyBatch(ctx, minimalValidApplyBatchRequest()) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected an error: NewUOW never succeeds") + } + if !IsSerializationError(err) { + t.Errorf("err = %v, want the last serialization failure itself: an exhausted retry budget, not ctx cancellation", err) + } + if elapsed > 1500*time.Millisecond { + t.Errorf("elapsed = %s, want well under %s: the marked path's ~1s floor should have governed, not the 2s outer ctx", elapsed, outerDeadline) + } + }) +} diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 0af723de1..8eff362fb 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -412,11 +412,65 @@ func (e *RefError) Error() string { func (e *RefError) Unwrap() error { return ErrValidation } // MaxApplyBatchItems is the largest request this role accepts. It is a FLAT -// bound rather than a cost model, and it is the bound the sibling batch roles -// already run under: a hundred creates and a hundred edges both land inside the -// server's ordinary request deadline today, so a per-item deadline extension -// would be new machinery answering a question nothing has asked. -const MaxApplyBatchItems = 100 +// bound rather than a cost model: ApplyBatchInTx issues roughly one round +// trip per item resolution plus a handful more per item's own writes, so the +// bound is really a bound on how long one request may hold a single write +// transaction open, not on batch semantics (which have no size in them). +// +// Raised from 100 to 1000: measured real-world plans — +// mol-normalized-issue-pr-work-v1, the largest observed shape at 356 items — +// already exceed the old 100-item bound, and 1000 is roughly 2.8x that +// measured max. The old doc comment claimed a hundred items "land inside the +// server's ordinary request deadline"; that is no longer the bound above 100 +// items. Instead, the HTTP edge (internal/httpapi) grants a request over 100 +// items a separate, EXTENDED whole-run budget (an operator-configured +// ceiling, `bd serve --large-apply-ceiling`, 5 minutes by default) rather +// than a value scaled by item count, takes a one-wide "large write" semaphore +// once a request exceeds the old 100-item threshold (so at most one +// oversized transaction holds a write connection at a time), and advertises +// all of this via the issues.batchApplyLarge capability token so a caller +// can tell whether the server it is talking to supports the raised cap +// before sending a request that size. +// +// The role itself still enforces no per-item cost model and still runs the +// WHOLE plan as ONE transaction no matter its size — raising this bound does +// not, and must never, change ApplyBatchInTx into something that chunks a +// plan into multiple transactions. A request either commits in full or +// changes nothing. +const MaxApplyBatchItems = 1000 + +// extendedRetryBudgetKey is the unexported key WithExtendedRetryBudget and +// HasExtendedRetryBudget share. +type extendedRetryBudgetKey struct{} + +// WithExtendedRetryBudget marks ctx as belonging to a large batch-apply whose +// commit-retry loop should be allowed to scale with the request's own +// extended run budget, instead of the ordinary fixed retry ceiling every +// other write uses. +// +// This exists because a BatchApplier implementation's commit-retry budget +// (internal/storage/uow's DefaultTxRetryMaxElapsed, 15s) must NOT change for +// an ordinary request just because deriving it from "whatever's left on +// ctx.Deadline()" would be simpler: an ordinary request's context also +// carries a real deadline (internal/httpapi's requestDeadline, 60s), and +// naively extending the retry budget to match ANY sufficiently long +// remaining deadline silently changed ordinary requests from a 15s retry +// ceiling to up to 60s — a regression an earlier version of this fix +// introduced. Marking the context explicitly, only on the one path +// (internal/httpapi's admitLargeApply) that intentionally grants a request an +// extended run budget, keeps the two concerns independent: an implementation +// reads HasExtendedRetryBudget to decide WHETHER to look at the deadline at +// all, rather than inferring intent from the deadline's mere size. +func WithExtendedRetryBudget(ctx context.Context) context.Context { + return context.WithValue(ctx, extendedRetryBudgetKey{}, true) +} + +// HasExtendedRetryBudget reports whether ctx was marked by +// WithExtendedRetryBudget. +func HasExtendedRetryBudget(ctx context.Context) bool { + marked, _ := ctx.Value(extendedRetryBudgetKey{}).(bool) + return marked +} // BatchApplier describes applying a HETEROGENEOUS list of graph mutations as // ONE durable act, and — like Lifecycle, BatchCreator, BatchCloser and From 20c198f76d55e168247f6805c68558d3fbb2a116 Mon Sep 17 00:00:00 2001 From: Eddie the Engineer Date: Thu, 1 Oct 2026 00:16:34 +0000 Subject: [PATCH 3/5] fix(httpapi,serve): wire the large-apply retry budget; repair stale pins Review fixes for the large batchApply envelope, on top of the test repairs the PR needed against main. The extended commit-retry budget was never switched on. internal/storage/uow's retryTxBudget lets a large apply keep retrying a commit-time conflict past the ordinary 15s ceiling only when its context carries issueops.WithExtendedRetryBudget's marker, and the docs on both sides say admitLargeApply sets it, but nothing did: the marker's only callers were the uow tests, which mark their contexts by hand. A 1000-item apply that hit a conflict still gave up after 15s with most of its 5-minute ceiling unused. admitLargeApply now marks the run context it builds, and TestApplyBatchMarksOnlyALargeApplyForTheExtendedRetryBudget pins the seam over HTTP: marked at 101 and 1000 items, unmarked at 100. Removing the mark fails the 101 and 1000 cases; marking every request in handleApplyBatch fails the 100 case. Smaller fixes from the same review: - `bd serve --large-apply-ceiling` repeated the 5m default as a literal of its own, and since bd serve always passes a positive ceiling, that literal, not httpapi's, was the default every deployment ran. Export httpapi.DefaultLargeApplyCeiling, default the flag to it, and pin the flag's 5m0s default in TestServeFlags. - admitLargeApply's release cancels the run context before clearing s.largeApplyDeadline, the order acquireLargeApply's doc states; it cleared the deadline first. - Drop the batchfixtures test's dead t.Skipf, which waited for the cap to reach 1000, and correct a graph-counter comment that still cited the 100-item cap and the TestServeFlags doc line that called the no-argument server unchanged. The PR raises issueops.MaxApplyBatchItems from 100 to 1000 and adds `bd serve --large-apply-ceiling`, but two base-owned tests still pinned the old surface and went red on every lane that runs them (Bazel / test, Test (macos-latest)): - TestServeFlags pins the exact `bd serve` flag list; add large-apply-ceiling. - TestPlanApplyBatchRefusesAnUnusableRequest/too_many_items built 101 items and expected "at most 100 items", a request the 1000 cap now plans successfully. Size the over-cap request and its message from issueops.MaxApplyBatchItems, as TestPlanApplyBatchAcceptsTheBoundary already does for the other side of the cap; the cap's value stays pinned by TestCapBatchApplyLargeTiesAllThreeLimits. The PR's own TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout also went red in PR Core (-race, -p 4 on a 4-vCPU runner): all eight small requests took 160-165ms against a 100ms bound. The stall it guards against lasts semTimeout (5s), while a small request can still queue briefly behind the refused large requests, each holding a general sem slot while its body decodes. Bound it at a second, the "want well under a second" its sibling TestLargeApplyQueueDoesNotStarveOrdinaryRequests already uses. Pinned to four cores beside 24 busy loops, small requests measured 183ms mean and 338ms max, and the 1s bound passed 80/80 runs; with the one-waiter gate in acquireLargeApply disabled the test still fails 3/3, on its 2s never-completed check. Co-Authored-By: Claude Opus 5.5 --- cmd/bd/serve.go | 2 +- cmd/bd/serve_test.go | 13 ++-- internal/httpapi/large_batch_apply_test.go | 70 ++++++++++++++++--- internal/httpapi/server.go | 32 ++++++--- internal/storage/batch_apply_test.go | 8 +-- .../batchfixtures/batchfixtures_test.go | 13 +--- .../large_batch_apply_measure_test.go | 4 +- issueops/graphcounter.go | 6 +- 8 files changed, 107 insertions(+), 41 deletions(-) diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index e8bbf2042..d27382bbd 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -176,7 +176,7 @@ func registerServeFlags(cmd *cobra.Command) { "Serve a non-loopback bind with NO authentication. Every peer that can reach the address gets full read and claim access") cmd.Flags().StringArrayVar(&serveAllowedHosts, "allowed-host", nil, "Additional Host header value to answer to, e.g. a service DNS name. Repeatable; matched exactly, with no wildcards") - cmd.Flags().DurationVar(&serveLargeApplyCeiling, "large-apply-ceiling", 5*time.Minute, + cmd.Flags().DurationVar(&serveLargeApplyCeiling, "large-apply-ceiling", httpapi.DefaultLargeApplyCeiling, "Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md") } diff --git a/cmd/bd/serve_test.go b/cmd/bd/serve_test.go index ea980a4d1..d9032699e 100644 --- a/cmd/bd/serve_test.go +++ b/cmd/bd/serve_test.go @@ -29,7 +29,10 @@ import ( // deliberately, rather than arriving as one nobody designed. // // Every flag added here defaults to the behavior that existed without it, so -// `bd serve` with no arguments is the same server it has always been. +// a flag's default is never what changes `bd serve` with no arguments from +// one release to the next. For --large-apply-ceiling that behavior is +// httpapi.DefaultLargeApplyCeiling, the ceiling the server applies when none +// is configured. func TestServeFlags(t *testing.T) { var got []string serveCmd.Flags().VisitAll(func(f *pflag.Flag) { got = append(got, f.Name) }) @@ -37,18 +40,20 @@ func TestServeFlags(t *testing.T) { want := []string{ "addr", "allow-non-loopback", "allowed-host", "auth-token-file", - "insecure-no-auth", + "insecure-no-auth", "large-apply-ceiling", } if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("bd serve flags = %v, want %v", got, want) } // The defaults ARE the compatibility promise: no token file, no extra - // hosts, and no waiver. + // hosts, no waiver, and the large-apply ceiling that SERVE_RUNBOOK.md + // sizes an orchestrator's stop grace against. for _, tc := range []struct{ flag, want string }{ {"auth-token-file", ""}, {"insecure-no-auth", "false"}, {"allowed-host", "[]"}, + {"large-apply-ceiling", "5m0s"}, } { f := serveCmd.Flags().Lookup(tc.flag) if f == nil { @@ -522,7 +527,7 @@ func withServeFlags(t *testing.T) { serveAddr, serveAllowNonLoopback = "127.0.0.1:0", false serveAuthTokenFile, serveInsecureNoAuth = "", false serveAllowedHosts = nil - serveLargeApplyCeiling = 5 * time.Minute + serveLargeApplyCeiling = httpapi.DefaultLargeApplyCeiling } func serveTokenFile(t *testing.T) string { diff --git a/internal/httpapi/large_batch_apply_test.go b/internal/httpapi/large_batch_apply_test.go index e80c4cec4..d3dca4af2 100644 --- a/internal/httpapi/large_batch_apply_test.go +++ b/internal/httpapi/large_batch_apply_test.go @@ -96,8 +96,8 @@ func TestCapBatchApplyLargeTiesAllThreeLimits(t *testing.T) { if largeApplyItemThreshold != wantThreshold { t.Errorf("largeApplyItemThreshold = %d, want %d", largeApplyItemThreshold, wantThreshold) } - if defaultLargeApplyCeiling != wantDefaultCeiling { - t.Errorf("defaultLargeApplyCeiling = %s, want %s", defaultLargeApplyCeiling, wantDefaultCeiling) + if DefaultLargeApplyCeiling != wantDefaultCeiling { + t.Errorf("DefaultLargeApplyCeiling = %s, want %s", DefaultLargeApplyCeiling, wantDefaultCeiling) } if !slices.Contains(behaviorCapabilities, CapBatchApplyLarge) { t.Fatal("CapBatchApplyLarge is not advertised in behaviorCapabilities; a client cannot discover this envelope") @@ -429,8 +429,12 @@ func TestLargeApplyQueueDoesNotStarveOrdinaryRequests(t *testing.T) { // a semTimeout long enough that the old bounded-but-unbounded-waiter-count // behavior would visibly stall small requests, 16 concurrent large requests // racing 8 concurrent small ones must never make a small request wait on the -// large-apply contention at all — each small one completes in well under -// 100ms, not "eventually, once a wait times out". +// large-apply contention at all — each small one completes in well under a +// second, not "eventually, once a wait times out" (semTimeout, 5s). The bound +// is a stall detector, not a latency benchmark: a small request can still +// queue briefly behind the refused large requests, each holding a general sem +// slot while its body decodes, and under -race on a loaded 4-vCPU CI runner +// that measured ~160ms. func TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout(t *testing.T) { applier := &controlledApplier{gate: make(chan struct{})} gateClosed := false @@ -479,8 +483,8 @@ func TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout(t *testing.T) { if res.resp.StatusCode != http.StatusOK { t.Errorf("small request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) } - if res.elapsed > 100*time.Millisecond { - t.Errorf("small request took %s, want well under 100ms: a waiter for the large-apply slot must never hold a general sem slot long enough to stall a small request", res.elapsed) + if res.elapsed > time.Second { + t.Errorf("small request took %s, want well under a second: a waiter for the large-apply slot must never hold a general sem slot long enough to stall a small request", res.elapsed) } case <-time.After(2 * time.Second): t.Fatalf("small request %d never completed — this is the starvation bug item 1 fixes", i) @@ -710,21 +714,34 @@ func TestDrainBudget(t *testing.T) { } // deadlineCapturingApplier records the ctx.Deadline() each call actually -// received, so an HTTP-level test can check what budget handleApplyBatch -// handed the role without a real backend or a sleep-based race. +// received, and whether that ctx carried issueops' extended-retry-budget +// marker, so an HTTP-level test can check what budget handleApplyBatch handed +// the role without a real backend or a sleep-based race. type deadlineCapturingApplier struct { mu sync.Mutex deadlines []time.Time + marked []bool } func (d *deadlineCapturingApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (issueops.ApplyBatchResult, error) { dl, _ := ctx.Deadline() d.mu.Lock() d.deadlines = append(d.deadlines, dl) + d.marked = append(d.marked, issueops.HasExtendedRetryBudget(ctx)) d.mu.Unlock() return issueops.ApplyBatchResult{Items: make([]issueops.ItemResult, len(req.Items))}, nil } +func (d *deadlineCapturingApplier) lastMarked(t *testing.T) bool { + t.Helper() + d.mu.Lock() + defer d.mu.Unlock() + if len(d.marked) == 0 { + t.Fatal("ApplyBatch was never called") + } + return d.marked[len(d.marked)-1] +} + func (d *deadlineCapturingApplier) last(t *testing.T) time.Time { t.Helper() d.mu.Lock() @@ -806,6 +823,43 @@ func TestApplyBatchDeadlineExceedsTheOrdinaryRequestDeadlineWhenConfiguredTo(t * } } +// TestApplyBatchMarksOnlyALargeApplyForTheExtendedRetryBudget pins the seam +// between the two halves of the large-apply retry budget. +// internal/storage/uow's retryTxBudget lets a commit-time conflict keep +// retrying past the ordinary 15s ceiling only when ctx carries +// issueops.WithExtendedRetryBudget's marker, and admitLargeApply is the one +// place that sets it. The uow tests mark their contexts by hand, and the +// deadline tests above never look at the marker, so without this pin the +// extension could be left unwired with every other test green. The case at +// the threshold matters as much as the ones over it: marking an ordinary +// request would raise its retry ceiling from 15s to nearly its whole 60s +// deadline, the regression the marker exists to prevent. +func TestApplyBatchMarksOnlyALargeApplyForTheExtendedRetryBudget(t *testing.T) { + applier := &deadlineCapturingApplier{} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier})) + + cases := []struct { + name string + items int + want bool + }{ + {"at threshold (100)", largeApplyItemThreshold, false}, + {"just over threshold (101)", largeApplyItemThreshold + 1, true}, + {"at cap (1000)", issueops.MaxApplyBatchItems, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resp := ts.claim(t, batchApplyPath, createItemsBody("alice", tc.items)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + if got := applier.lastMarked(t); got != tc.want { + t.Errorf("%d items: issueops.HasExtendedRetryBudget(applier's ctx) = %t, want %t", tc.items, got, tc.want) + } + }) + } +} + // controlledApplier is a BatchApplier that blocks a call carrying more than // largeApplyItemThreshold items on a caller-controlled gate, so an HTTP-level // test can prove the semaphore serializes oversized requests without a real diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 2c4e8def5..da0d1cd89 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -92,7 +92,7 @@ const ( drainGrace = 5 * time.Second // uowCloseTimeout bounds the DETACHED close described on WithUOW. uowCloseTimeout = 5 * time.Second - // defaultLargeApplyCeiling is Config.LargeApplyCeiling's default: the + // DefaultLargeApplyCeiling is Config.LargeApplyCeiling's default: the // whole-run budget a POST issues:batchApply request over // largeApplyItemThreshold items gets once it holds the one-wide // large-apply slot (acquireLargeApply). It is an operator-configurable @@ -101,7 +101,12 @@ const ( // slot, whether it carries 101 items or 1000, because scaling it down for // a request well under the envelope's top end would narrow a caller's // budget for a reason unrelated to what it actually sent. - defaultLargeApplyCeiling = 5 * time.Minute + // + // It is exported so `bd serve --large-apply-ceiling` defaults to this + // same value instead of repeating it: bd serve always passes a positive + // ceiling, so a second literal there, not this constant, would be the + // default every bd serve deployment actually runs with. + DefaultLargeApplyCeiling = 5 * time.Minute ) // Pool limits for the provider's *sql.DB. The semaphore bounds handlers, not @@ -167,10 +172,10 @@ type Config struct { AllowedHosts []string // LargeApplyCeiling bounds how long POST /v0/beads/issues:batchApply may // hold a write transaction for a request over largeApplyItemThreshold - // items (batch_apply.go). Zero takes defaultLargeApplyCeiling (5 + // items (batch_apply.go). Zero takes DefaultLargeApplyCeiling (5 // minutes). It is wired to `bd serve --large-apply-ceiling`. // - // It is a CEILING, not a target — see defaultLargeApplyCeiling for why + // It is a CEILING, not a target — see DefaultLargeApplyCeiling for why // every request over the threshold gets the whole budget rather than a // value scaled to its item count. Raising or lowering it is an // operational decision, not a wire change: it does not move @@ -448,7 +453,7 @@ type Server struct { // sem the instant enough large applies queued up. largeApplySem chan struct{} // largeApplyCeiling is the resolved Config.LargeApplyCeiling (defaulted - // via orDefault to defaultLargeApplyCeiling). acquireLargeApply builds + // via orDefault to DefaultLargeApplyCeiling). acquireLargeApply builds // each run's deadline from this, fresh, at the moment the slot is // acquired — never before, so time spent queued is never charged against // the run itself. @@ -631,7 +636,7 @@ func Listen(cfg Config) (*Server, error) { semTimeout: semAcquireTimeout, semWarn: saturationWarn, largeApplySem: make(chan struct{}, 1), - largeApplyCeiling: orDefault(cfg.LargeApplyCeiling, defaultLargeApplyCeiling), + largeApplyCeiling: orDefault(cfg.LargeApplyCeiling, DefaultLargeApplyCeiling), closing: make(chan struct{}), maxWatchStreams: maxWatchStreams, @@ -1548,6 +1553,12 @@ func (s *Server) acquireLargeApply(ctx context.Context) (runCtx context.Context, // builds the run's own detached, freshly-timed context and stamps // s.largeApplyDeadline for Serve's graceful-drain budget to read. // +// The run context is also marked with issueops.WithExtendedRetryBudget, and +// this is the only place that marks one: the marker, not the deadline's size, +// is what lets a BatchApplier's commit-retry loop scale to the extended +// budget instead of stopping at its ordinary fixed ceiling. An ordinary +// request never reaches this function, so it stays unmarked. +// // It re-checks s.closing AFTER taking the slot and storing the deadline, // because a select with multiple ready cases (largeApplySem and s.closing // both ready at once, in acquireLargeApply above) picks pseudo-randomly: the @@ -1557,12 +1568,15 @@ func (s *Server) acquireLargeApply(ctx context.Context) (runCtx context.Context, // request this budget was sized for is the only one that can still be // holding the slot" invariant true. func (s *Server) admitLargeApply(ctx context.Context) (runCtx context.Context, release func(), err error) { - runCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), orDefault(s.largeApplyCeiling, defaultLargeApplyCeiling)) + runCtx, cancel := context.WithTimeout( + issueops.WithExtendedRetryBudget(context.WithoutCancel(ctx)), + orDefault(s.largeApplyCeiling, DefaultLargeApplyCeiling), + ) deadline, _ := runCtx.Deadline() s.largeApplyDeadline.Store(&deadline) release = func() { - s.largeApplyDeadline.Store(nil) cancel() + s.largeApplyDeadline.Store(nil) <-s.largeApplySem } @@ -2236,7 +2250,7 @@ func (s *Server) logStartup() { // separate from the ordinary "deadline" above: an operator reading // this line must not conclude every request here is bounded by the // 60s "deadline" figure just because that is the only one printed. - "large_apply_ceiling", orDefault(s.largeApplyCeiling, defaultLargeApplyCeiling).String(), + "large_apply_ceiling", orDefault(s.largeApplyCeiling, DefaultLargeApplyCeiling).String(), } // The pool bounds are this server's, applied to the provider above. On the // roles source there is no pool here to bound, and printing the numbers diff --git a/internal/storage/batch_apply_test.go b/internal/storage/batch_apply_test.go index 67b5cefdc..33b148958 100644 --- a/internal/storage/batch_apply_test.go +++ b/internal/storage/batch_apply_test.go @@ -58,9 +58,9 @@ func int64Ptr(v int64) *int64 { return &v } // every one of them is knowable from the request alone, so a request refused // here has provably touched no database. func TestPlanApplyBatchRefusesAnUnusableRequest(t *testing.T) { - hundredAndOne := make([]issueops.ApplyItem, 101) - for i := range hundredAndOne { - hundredAndOne[i] = planApplyCreate("") + overCap := make([]issueops.ApplyItem, issueops.MaxApplyBatchItems+1) + for i := range overCap { + overCap[i] = planApplyCreate("") } for _, test := range []struct { @@ -70,7 +70,7 @@ func TestPlanApplyBatchRefusesAnUnusableRequest(t *testing.T) { }{ {"no actor", issueops.ApplyBatchRequest{Items: []issueops.ApplyItem{planApplyCreate("")}}, "requires an actor"}, {"no items", issueops.ApplyBatchRequest{Actor: "planner"}, "at least one item"}, - {"too many items", issueops.ApplyBatchRequest{Actor: "planner", Items: hundredAndOne}, "at most 100 items"}, + {"too many items", issueops.ApplyBatchRequest{Actor: "planner", Items: overCap}, fmt.Sprintf("at most %d items", issueops.MaxApplyBatchItems)}, {"no payload", planApplyRequest(issueops.ApplyItem{Kind: issueops.ItemCreate}), "exactly one payload"}, { "two payloads", diff --git a/internal/storage/batchfixtures/batchfixtures_test.go b/internal/storage/batchfixtures/batchfixtures_test.go index fa184b5ef..ab7bb7758 100644 --- a/internal/storage/batchfixtures/batchfixtures_test.go +++ b/internal/storage/batchfixtures/batchfixtures_test.go @@ -83,16 +83,9 @@ func TestShapeItemCounts(t *testing.T) { // Every shape must validate and plan cleanly against the role's own // request-shape contract (ref ordering, key uniqueness, - // exactly-one-payload) whenever it fits under the role's item cap — - // a fixture that failed it would be measuring nothing. The 356- and - // 712-item shapes exceed issueops.MaxApplyBatchItems (100) before B1 - // raises it to 1000, so this check is skipped above the current cap - // rather than ordered against a B1 commit that may not have landed - // yet; B1's own tests cover the cap itself. - if len(tc.req.Items) > issueops.MaxApplyBatchItems { - t.Skipf("item count %d exceeds current issueops.MaxApplyBatchItems (%d); cap coverage lives in the B1 tests", - len(tc.req.Items), issueops.MaxApplyBatchItems) - } + // exactly-one-payload, the issueops.MaxApplyBatchItems cap) — a + // fixture that failed it would be measuring nothing, and one over + // the cap would be measuring a request no caller can send. if _, err := storage.PlanApplyBatch(tc.req); err != nil { t.Fatalf("PlanApplyBatch: %v", err) } diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go index 59f0e5574..5678b3d3c 100644 --- a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -250,7 +250,7 @@ var wallClockShapes = []struct { // on the embedded backend at 356 and 1000 items. It is deliberately NOT a // pass/fail regression gate — wall-clock time is host- and CI-runner- // sensitive in a way statement counts are not — but the numbers it logs are -// the real-data justification for defaultLargeApplyCeiling +// the real-data justification for httpapi.DefaultLargeApplyCeiling // (internal/httpapi/server.go, 5 minutes): both measured shapes must // complete in a small fraction of that budget for the ceiling to be // generous headroom rather than a number picked out of thin air. @@ -290,7 +290,7 @@ func TestLargeBatchApplyWallClock_Embedded(t *testing.T) { } elapsed := time.Since(start) - t.Logf("embedded wall-clock: %s items (%d actual) = %s (defaultLargeApplyCeiling headroom: %.1fx)", + t.Logf("embedded wall-clock: %s items (%d actual) = %s (DefaultLargeApplyCeiling headroom: %.1fx)", tc.name, len(plan.Items), elapsed, (5*time.Minute).Seconds()/elapsed.Seconds()) }) } diff --git a/issueops/graphcounter.go b/issueops/graphcounter.go index 01ea14058..17eae3594 100644 --- a/issueops/graphcounter.go +++ b/issueops/graphcounter.go @@ -80,9 +80,9 @@ type EdgeCountRequest struct { // be a limit invented for a caller that is not the one that needs it. The // bound belongs to the WIRE operation, where the request arrives from // somewhere else and the cost of an unbounded one is a stranger's — the - // split ApplyBatchRequest.Items makes explicitly by bounding at 100. The - // graph-counts wire slice set it: GET /v0/beads/dependencies:count bounds - // `issue_id` at minItems 1, maxItems 100. + // split ApplyBatchRequest.Items makes explicitly with MaxApplyBatchItems. + // The graph-counts wire slice set it: GET /v0/beads/dependencies:count + // bounds `issue_id` at minItems 1, maxItems 100. IDs []string // Direction is which edges the count is over, and it is REQUIRED. An empty From c3fa831888fff52a521bdca9937aaf1ed3c83f4c Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Thu, 1 Oct 2026 08:38:20 +0000 Subject: [PATCH 4/5] fix(uow): stop a doomed large-apply retry before ctx cuts it off retryTxBudget sizes maxElapsed once at ApplyBatch entry, but says nothing about how long an individual attempt takes. A slow first attempt (e.g. 157s of a large apply's commit) left little of the budget for a retry, which then got killed mid-transaction by ctx's own deadline instead of finishing and reporting its own outcome -- surfacing as context.DeadlineExceeded (isUnavailable excludes it -> a generic 500) instead of the exhausted-write-conflict result this loop already reports correctly (-> 503 + Retry-After). RunTxResultWithin now tracks the previous attempt's duration and, before starting a new one, stops early with the last real serialization error when time left on ctx is less than that duration plus a fixed headroom -- but only when publicops.HasExtendedRetryBudget(ctx) is set (the same marker retryTxBudget itself checks). Applying the look-ahead unconditionally regressed TestApplyBatchHonorsAnExtendedRetryBudgetFromContext's unmarked case: ordinary requests have cheap attempts and are meant to run until their own ctx deadline ends the loop, and the large-apply-sized 5s headroom was tripping that for any short-ctx caller with a near-instant failing attempt. Adds TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry (a 1.2s commit against a 2s marked ctx must stop after one attempt with its own serialization error, never DeadlineExceeded, and never touch a second unit of work) and its control, TestRunTxResultWithin_ShortAttemptStillRetries (a 50ms attempt against the same shape must still retry and succeed). Both mutation-checked against the look-ahead condition and its gate. Also updates the stale "items accepts at most 100 entries" prose on the applyBatch endpoint description (openapi.v0.yaml) to the large-envelope wording already used elsewhere in the spec: 1000-item cap behind the issues.batchApplyLarge capability, large-apply-ceiling deadline extension, <=100 unaffected. dependencies:count's own "at most 100" lines are unrelated and untouched. Co-Authored-By: Claude Sonnet 5 --- internal/httpapi/spec/openapi.v0.yaml | 19 ++++-- internal/storage/uow/tx.go | 59 ++++++++++++++++- internal/storage/uow/tx_test.go | 95 ++++++++++++++++++++++++++- 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 38bc0c0a2..71d23b419 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -3841,12 +3841,19 @@ paths: ## What lands, and what it costs - `items` accepts at most 100 entries. The cap bounds how long one request - may hold a write transaction — it is not a statement about batch - semantics, and it is the bound the sibling batch operations already run - under. Split a larger plan; each request is atomic on its own, but note - that splitting it changes what the end gate can see, since the gate runs - over one request at a time. + `items` accepts at most 1000 entries (raised from 100; see the + `issues.batchApplyLarge` capability token on `ContextResponse`). The + cap bounds how long one request may hold a write transaction — it is + not a statement about batch semantics. A request over 100 items also + EXTENDS the effective request deadline to an operator-configured + ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) + instead of the ordinary small-request deadline, so a plan that + genuinely needs minutes to apply is not cut off mid-commit; a request + at or under 100 items keeps exactly the deadline — and the sibling + batch operations' bound — it has always had. Split a larger plan if + you must; each request is atomic on its own, but note that splitting + it changes what the end gate can see, since the gate runs over one + request at a time. ONE HISTORY ENTRY IS RECORDED FOR THE WHOLE REQUEST, attributed to diff --git a/internal/storage/uow/tx.go b/internal/storage/uow/tx.go index ba7e6af6e..35cb0a94c 100644 --- a/internal/storage/uow/tx.go +++ b/internal/storage/uow/tx.go @@ -8,6 +8,7 @@ import ( "github.com/steveyegge/beads/internal/storage/domain/db" "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" ) type Tx interface { @@ -86,11 +87,63 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap bo.InitialInterval = txRetryInitialInterval bo.MaxElapsedTime = maxElapsed + // lastAttemptDuration and lastErr let the loop decide, BEFORE starting a + // new attempt, whether there is realistically enough time left to finish + // one. maxElapsed is a BUDGET fixed once at the top of the call; it says + // nothing about how long any individual attempt actually takes. A slow + // attempt (a large apply's commit can run well past a minute) can lose + // the race against ctx's own absolute deadline mid-transaction: the retry + // loop would start another attempt because elapsed-so-far is still under + // maxElapsed, that attempt gets killed by ctx cancellation partway + // through, and the caller sees context.DeadlineExceeded — classified a + // generic 500 — instead of the exhausted-write-conflict outcome this loop + // already reports correctly when it gives up for its own reasons + // (classified a retryable 503). Comparing the time left to the PREVIOUS + // attempt's own duration is a cheap, evidence-based stand-in for + // predicting the next one, since a Dolt commit's cost is driven by the + // same plan on every attempt. + // + // This look-ahead is gated on publicops.HasExtendedRetryBudget(ctx), the + // same marker retryTxBudget itself checks, and for the same reason: an + // ordinary (unmarked) caller's attempts are cheap, and its ctx deadline + // is meant to be the thing that ends the loop — see + // TestApplyBatchHonorsAnExtendedRetryBudgetFromContext's "unmarked" + // case. Applying this check unconditionally would fire for those callers + // too: retryTxAttemptHeadroom is sized for a large apply's multi-minute + // commits, so for a near-instant failing attempt it would treat almost + // any ctx with less than retryTxAttemptHeadroom left as too tight, + // cutting the loop short well before ctx's own deadline and turning what + // should be ctx's DeadlineExceeded into a serialization error instead. + // Scoping this to the extended-budget (large apply) path keeps ordinary + // callers' retry-until-ctx-ends-it behavior exactly as before. + var lastAttemptDuration time.Duration + var lastErr error + haveAttempted := false + lookAhead := publicops.HasExtendedRetryBudget(ctx) + err := backoff.Retry(func() error { + if haveAttempted && lookAhead { + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < lastAttemptDuration+retryTxAttemptHeadroom { + // A retry this long would almost certainly be cut off by + // ctx's own deadline rather than finish and report its own + // outcome. Stop now and surface the LAST attempt's real + // error — still a retryable conflict — rather than start a + // doomed attempt that ends in ctx.Err(). + return backoff.Permanent(lastErr) + } + } + attemptStart := time.Now() + fail := func(err error) error { + lastAttemptDuration = time.Since(attemptStart) + lastErr = err + haveAttempted = true + return err + } + uw, err := p.NewUOW(ctx) if err != nil { if isSerializationError(err) { - return err + return fail(err) } return backoff.Permanent(err) } @@ -99,7 +152,7 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap r, commitMsg, err := work(ctx, uw) if err != nil { if isSerializationError(err) { - return err + return fail(err) } return backoff.Permanent(err) } @@ -115,7 +168,7 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap return nil } if isSerializationError(err) { - return err + return fail(err) } return backoff.Permanent(err) } diff --git a/internal/storage/uow/tx_test.go b/internal/storage/uow/tx_test.go index 61bd9d013..0e3f3ac8a 100644 --- a/internal/storage/uow/tx_test.go +++ b/internal/storage/uow/tx_test.go @@ -10,11 +10,17 @@ import ( "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage/domain" + publicops "github.com/steveyegge/beads/issueops" ) // mockUnitOfWork implements UnitOfWork for testing type mockUnitOfWork struct { - commitErr error + commitErr error + // commitDelay, when set, is slept inside Commit before returning + // commitErr — lets a test simulate an attempt that takes real wall-clock + // time to fail (a slow commit racing a context deadline), as opposed to + // commitErr alone, which fails instantly. + commitDelay time.Duration commitCount int closed bool configUseCase domain.ConfigUseCase @@ -36,6 +42,9 @@ func (m *mockUnitOfWork) Close(ctx context.Context) { func (m *mockUnitOfWork) Commit(ctx context.Context, message string) error { m.commitCount++ + if m.commitDelay > 0 { + time.Sleep(m.commitDelay) + } return m.commitErr } @@ -503,6 +512,90 @@ func TestRunTx_NewUOWError(t *testing.T) { } } +// TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry pins the fix for +// the gap bee's live 1000-item run exposed: retryTxBudget / maxElapsed is a +// BUDGET fixed once at the top of the call, and says nothing about how long +// any one attempt actually takes. Before this fix, a slow failing attempt +// (its commit loses Dolt's merge after running most of the remaining ctx +// deadline) still triggered a retry whenever elapsed-so-far was under +// maxElapsed — and that retry then got killed mid-transaction by ctx's own +// deadline, surfacing as context.DeadlineExceeded (an internal-httpapi +// isUnavailable() miss -> a generic 500) instead of the exhausted-write- +// conflict outcome this loop already reports correctly when IT gives up +// (uow.IsSerializationError -> httpapi maps that to 503 + Retry-After). +// +// The first attempt's commit sleeps 1.2s (over half the 2s ctx) before +// failing with a retryable deadlock; with retryTxAttemptHeadroom (5s) added +// on top, the ~0.8s left after that attempt can never cover another attempt +// of similar cost, so the loop must stop WITHOUT trying uw2 at all and return +// uw1's own serialization error. +// +// The look-ahead this pins is gated on publicops.HasExtendedRetryBudget, the +// same marker retryTxBudget checks (see that gate's comment in tx.go), so +// this ctx must carry it — an unmarked ctx would skip the new check entirely +// and this test would regress to asserting old, unfixed behavior. +func TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry(t *testing.T) { + uw1 := &mockUnitOfWork{commitErr: newMySQLError(1213), commitDelay: 1200 * time.Millisecond} + uw2 := &mockUnitOfWork{} + provider := &mockUnitOfWorkProvider{uows: []*mockUnitOfWork{uw1, uw2}} + + ctx, cancel := context.WithTimeout(publicops.WithExtendedRetryBudget(context.Background()), 2*time.Second) + defer cancel() + + _, err := RunTxResultWithin(ctx, provider, 10*time.Second, func(ctx context.Context, uw UnitOfWork) (struct{}, string, error) { + return struct{}{}, "slow attempt probe", nil + }) + + if err == nil { + t.Fatal("expected an error: the only attempt tried fails on commit") + } + if !IsSerializationError(err) { + t.Errorf("err = %v, want uw1's own serialization error (-> 503 Retry-After); a ctx-cancellation error here would be the regression (-> a generic 500)", err) + } + if errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v wraps context.DeadlineExceeded: a doomed second attempt was started and got cut off by ctx instead of being skipped", err) + } + if provider.newUOWCalls != 1 { + t.Errorf("NewUOW calls = %d, want exactly 1: a second attempt was started even though it could not have finished before ctx's deadline", provider.newUOWCalls) + } + if uw2.commitCount != 0 { + t.Errorf("uw2.commitCount = %d, want 0: uw2 must never be reached", uw2.commitCount) + } +} + +// TestRunTxResultWithin_ShortAttemptStillRetries is +// TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry's control: when the +// failing attempt is fast relative to what's left on ctx, the new +// look-before-you-retry check must not block the retry that would otherwise +// succeed. +func TestRunTxResultWithin_ShortAttemptStillRetries(t *testing.T) { + uw1 := &mockUnitOfWork{commitErr: newMySQLError(1213), commitDelay: 50 * time.Millisecond} + uw2 := &mockUnitOfWork{} + provider := &mockUnitOfWorkProvider{uows: []*mockUnitOfWork{uw1, uw2}} + + // 8s comfortably exceeds retryTxAttemptHeadroom (5s) by more than the 50ms + // first attempt costs, so the real-time budget this test takes is + // governed by how fast the two attempts actually run (well under a + // second), not by this deadline. Marked for the same reason as the slow- + // attempt test above: the look-ahead only runs when this marker is set. + ctx, cancel := context.WithTimeout(publicops.WithExtendedRetryBudget(context.Background()), 8*time.Second) + defer cancel() + + _, err := RunTxResultWithin(ctx, provider, 15*time.Second, func(ctx context.Context, uw UnitOfWork) (struct{}, string, error) { + return struct{}{}, "short attempt probe", nil + }) + + if err != nil { + t.Fatalf("expected no error after retry, got %v", err) + } + if provider.newUOWCalls < 2 { + t.Errorf("NewUOW calls = %d, want at least 2: a short failed attempt must still be retried", provider.newUOWCalls) + } + if uw2.commitCount != 1 { + t.Errorf("uw2.commitCount = %d, want 1", uw2.commitCount) + } +} + func TestRunTx_WorkSerializationErrorRetries(t *testing.T) { // Work function itself returns serialization error (not commit) uw1 := &mockUnitOfWork{} From 7d8d858427becdfa03fbe8a96a352890e764b7ef Mon Sep 17 00:00:00 2001 From: Eddie the Engineer Date: Thu, 1 Oct 2026 11:02:00 +0000 Subject: [PATCH 5/5] fix(ci,serve): fit the 1000-item cap's tests in the storage shard budget Review fixes for the relaunched #7051, iteration 1. Raising issueops.MaxApplyBatchItems from 100 to 1000 made the base-owned TestBatchApplyContract take 604s under -race, against 97s at the old cap (main run 35862663577). It hashed onto the legacy test-embedded-storage shard 2/5, beside ~5m of other tests, and that shard hit its 15m budget twice (PR Risk run 36807191489). PR Risk no longer runs the legacy tier on same-repo PRs (#7071), but Main's push run and fork PRs still do, so the merge would have carried the timeout to Main. Raising the jobs' timeout-minutes alone would not fix it: the shard script passes the binary its own -test.timeout=15m. So: - embedded-storage-test-shard.sh passes -test.timeout=20m (and its go test fallback -timeout 20m), as embedded-test-shard.sh does for cmd/bd. - test-embedded-storage in main.yml and pr-risk.yml: timeout-minutes 15 -> 25, the script's 20m plus setup. - embeddeddolt_embedded_test appends -test.timeout=19m, as bd_embedded_test does, so Go's timeout, which dumps every goroutine, fires before rbe-west's 1200s action kill, which does not. TestBazelEmbeddedJobMirrorsEmbeddedTier now checks the override on both sharded targets instead of only cmd/bd's, and the PR Risk coverage test pins the storage target's new args. - embedded-storage-test-shards.txt places TestBatchApplyContract on shard 4, which has ~3m of other tests, and the PR's two large-batch tests, TestLargeBatchApplyStatementCounts_Embedded (247s) and TestLargeBatchApplyWallClock_Embedded (261s), on shards 2 and 5. From the PR run's per-test times, the heaviest shard projects to ~13m, down from shard 2's 15m-plus. Also correct the stop-grace guidance. For an in-flight large apply, Serve's drain waits out the apply's remaining deadline plus drainGrace (5s), so the orchestrator's stop grace must be at least --large-apply-ceiling plus 5s, as Serve's own doc says. The runbook, the flag help, the LargeApplyCeiling field doc and the CHANGELOG entry asked for the ceiling alone. Co-Authored-By: Claude Opus 5.5 --- .bazelrc | 9 ++++---- .../scripts/embedded-storage-test-shard.sh | 8 +++++-- .../scripts/embedded-storage-test-shards.txt | 12 ++++++++++ .github/workflows/main.yml | 7 +++++- .github/workflows/pr-risk.yml | 7 +++++- CHANGELOG.md | 20 ++++++++--------- cmd/bd/serve.go | 2 +- engdocs/SERVE_RUNBOOK.md | 22 +++++++++---------- internal/httpapi/server.go | 8 +++---- internal/storage/embeddeddolt/BUILD.bazel | 9 +++++--- scripts/ci_workflow_test.go | 16 +++++++++----- scripts/pr_risk_bazel_coverage_test.go | 2 +- 12 files changed, 77 insertions(+), 45 deletions(-) diff --git a/.bazelrc b/.bazelrc index 2ccf130de..c3d60537f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -89,11 +89,10 @@ test:embedded --keep_going test:embedded --test_summary=terse # The variants are timeout = "eternal" because the jobs' Go timeouts are past # "long"; rbe-west rejects any action timeout over 1200s, so eternal means -# 1200s here, and the effective limits are: storage shards, the job's -# -test.timeout=15m; cmd shards and conformance core/audit, -test.timeout=19m -# (the jobs pass 20m and 30m), so that Go's timeout, which dumps every -# goroutine, fires before Bazel's 1200s kill, which does not (see the BUILD -# targets). +# 1200s here, and the effective limit of every variant is -test.timeout=19m +# (the storage and cmd shard jobs pass 20m, conformance core/audit 30m), so +# that Go's timeout, which dumps every goroutine, fires before Bazel's 1200s +# kill, which does not (see the BUILD targets). test:embedded --test_timeout=-1,-1,-1,1200 # The jobs pass no -parallel: GOMAXPROCS on a 4-vCPU runner. test:embedded --test_arg=-test.parallel=4 diff --git a/.github/scripts/embedded-storage-test-shard.sh b/.github/scripts/embedded-storage-test-shard.sh index 2f78a7883..0d6968e86 100755 --- a/.github/scripts/embedded-storage-test-shard.sh +++ b/.github/scripts/embedded-storage-test-shard.sh @@ -140,13 +140,17 @@ if [ "${BEADS_TEST_SHARD_LIST_ONLY:-}" = "1" ]; then exit 0 fi +# 20m, as embedded-test-shard.sh: under -race, TestBatchApplyContract alone +# takes ~10m at the 1000-item issueops.MaxApplyBatchItems cap, and the +# heaviest shard projects to ~13m (see embedded-storage-test-shards.txt). +# The jobs' timeout-minutes (25) leave room for setup after this fires. if [ -x "$STORAGE_BINARY" ]; then - exec "$STORAGE_BINARY" -test.v -test.count=1 -test.timeout=15m \ + exec "$STORAGE_BINARY" -test.v -test.count=1 -test.timeout=20m \ -test.run "$RUN_REGEX" \ "$@" else echo "Warning: pre-built test binary not found at $STORAGE_BINARY, falling back to go test" - exec go test -tags=gms_pure_go -v -race -count=1 -timeout 15m \ + exec go test -tags=gms_pure_go -v -race -count=1 -timeout 20m \ -run "$RUN_REGEX" \ "$@" \ ./internal/storage/embeddeddolt/ diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 1327de7a3..cfd0dbe78 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -15,10 +15,20 @@ # excluded — it runs in the separate test-embedded-conformance job, which # shards by sub-test path (core/audit), not by top-level function name, and # is a different mechanism from this manifest. +# +# Three tests are placed by measured -race duration instead (PR Risk run +# 36807191489, at the 1000-item issueops.MaxApplyBatchItems cap): +# TestBatchApplyContract (604s) on shard 4, which has ~3m of other tests, +# and TestLargeBatchApplyStatementCounts_Embedded (247s) and +# TestLargeBatchApplyWallClock_Embedded (261s) on shards 2 and 5. Hashed, +# TestBatchApplyContract shared shard 2 with ~5m of other tests and ran it +# past the job's old 15m timeout; placed so, the heaviest shard projects to +# ~13m. 5 1 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase 5 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase 5 3 TestAddDependency 5 4 TestAddLabel +5 4 TestBatchApplyContract 5 5 TestBuildDSN_NoDatabase 5 1 TestBuildDSN_SpacesInPath 5 2 TestBuildDSN_WithDatabase @@ -73,6 +83,8 @@ 5 1 TestHistory_NullTextColumns 5 2 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore 5 3 TestIsBlockedBatchEmbedded +5 2 TestLargeBatchApplyStatementCounts_Embedded +5 5 TestLargeBatchApplyWallClock_Embedded 5 4 TestLeaseLifecycleEmbedded 5 5 TestListWisps 5 1 TestOpenAfterCloseCreatesNewStore diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1045f28f3..8a07aba1a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1053,7 +1053,12 @@ jobs: needs: [detect-ci-tier, build-embedded] if: needs.detect-ci-tier.outputs.full_embedded == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item + # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s + # under -race (97s at the 100-item cap, main run 35862663577) and the + # shard hit 15m twice (PR Risk run 36807191489). The manifest now pins + # the heavy tests apart; the heaviest shard projects to ~13m. + timeout-minutes: 25 strategy: fail-fast: false matrix: diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index cc5c8c556..8bbb27591 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -265,7 +265,12 @@ jobs: needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item + # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s + # under -race (97s at the 100-item cap, main run 35862663577) and the + # shard hit 15m twice (run 36807191489). The manifest now pins the heavy + # tests apart; the heaviest shard projects to ~13m. + timeout-minutes: 25 strategy: fail-fast: false matrix: diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c2a3c4b..0b6a4941b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -614,16 +614,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 waits out an in-flight large apply up to its own deadline, but refuses any large apply still queued for the semaphore once shutdown begins, rather than extending the drain further. `bd serve`'s orchestrator stop grace must be at - least `--large-apply-ceiling`, or an external `SIGKILL` can still cut off an - in-flight large apply the drain would otherwise have waited out; see - `engdocs/SERVE_RUNBOOK.md`. A new `issues.batchApplyLarge` capability token - on `GET /v0/beads/context` lets a client learn whether a given server - accepts the raised envelope before sending a plan over the old 100-item - bound. Measured wall-clock for the whole-transaction apply (embedded Dolt, - build+commit only) backs the 5-minute default: a 356-item plan (the design - doc's primary measured shape) commits in ~12.6s (23.8x headroom under the - ceiling), and a 1000-item plan (the new cap) commits in ~57.5s (5.2x - headroom). + least `--large-apply-ceiling` plus 5 seconds, or an external `SIGKILL` can + still cut off an in-flight large apply the drain would otherwise have waited + out; see `engdocs/SERVE_RUNBOOK.md`. A new `issues.batchApplyLarge` + capability token on `GET /v0/beads/context` lets a client learn whether a + given server accepts the raised envelope before sending a plan over the old + 100-item bound. Measured wall-clock for the whole-transaction apply + (embedded Dolt, build+commit only) backs the 5-minute default: a 356-item + plan (the design doc's primary measured shape) commits in ~12.6s (23.8x + headroom under the ceiling), and a 1000-item plan (the new cap) commits in + ~57.5s (5.2x headroom). - **A long-running schema migration now says so instead of going quiet** ([#5997](https://github.com/gastownhall/beads/pull/5997)). Migrations are diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index d27382bbd..3110191da 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -177,7 +177,7 @@ func registerServeFlags(cmd *cobra.Command) { cmd.Flags().StringArrayVar(&serveAllowedHosts, "allowed-host", nil, "Additional Host header value to answer to, e.g. a service DNS name. Repeatable; matched exactly, with no wildcards") cmd.Flags().DurationVar(&serveLargeApplyCeiling, "large-apply-ceiling", httpapi.DefaultLargeApplyCeiling, - "Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md") + "Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long plus 5s, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md") } // serveOptions is the part of a server's configuration that depends on NEITHER diff --git a/engdocs/SERVE_RUNBOOK.md b/engdocs/SERVE_RUNBOOK.md index 5a21d9fd6..a6a04123f 100644 --- a/engdocs/SERVE_RUNBOOK.md +++ b/engdocs/SERVE_RUNBOOK.md @@ -323,19 +323,19 @@ unable to tell whether its write landed. budget rather than being cut off by it.** Such a request can legitimately still be running at up to `--large-apply-ceiling` (5 minutes by default) when shutdown begins, so the drain budget computed at shutdown start extends to that -request's own remaining deadline instead of the ordinary 20 seconds — the -in-flight write is waited out, not killed mid-commit. A large apply still -*queued* for the one-wide large-apply semaphore when shutdown begins is refused -with `503` immediately rather than being admitted or extending the drain -further. +request's own remaining deadline, plus 5 seconds to write its response back, +instead of the ordinary 20 seconds — the in-flight write is waited out, not +killed mid-commit. A large apply still *queued* for the one-wide large-apply +semaphore when shutdown begins is refused with `503` immediately rather than +being admitted or extending the drain further. **The orchestrator's stop grace (e.g. a Kubernetes `terminationGracePeriodSeconds`) -must be at least `--large-apply-ceiling`.** The drain logic above only runs -inside this process; an external `SIGKILL` after the orchestrator's own grace -period expires cuts the process off regardless of what the drain was waiting -for, which can still sever an in-flight large apply the drain would otherwise -have waited out. Size the grace period to the ceiling, not to the ordinary 20s -drain. +must be at least `--large-apply-ceiling` plus 5 seconds.** The drain logic above +only runs inside this process; an external `SIGKILL` after the orchestrator's +own grace period expires cuts the process off regardless of what the drain was +waiting for, which can still sever an in-flight large apply the drain would +otherwise have waited out. Size the grace period to the ceiling plus those 5 +seconds, not to the ordinary 20s drain. Journal streams are the exception, and they have to be: a held-open response would otherwise sit through the whole budget and then be killed anyway. They get diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index da0d1cd89..dfdff48f6 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -185,10 +185,10 @@ type Config struct { // issues.batchApplyLarge capability token would otherwise advertise a // budget it no longer honors. // - // The orchestrator's stop grace must be at least this long: Serve's drain - // extends its budget to match an in-flight large apply's deadline, but a - // SIGKILL from outside this process does not consult it. See - // engdocs/SERVE_RUNBOOK.md. + // The orchestrator's stop grace must be at least this long plus + // drainGrace (5s): Serve's drain extends its budget to an in-flight large + // apply's remaining deadline plus drainGrace, but a SIGKILL from outside + // this process does not consult it. See engdocs/SERVE_RUNBOOK.md. LargeApplyCeiling time.Duration // Provider is where every database-touching handler opens its one unit of // work per request. diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 8d6f09dfc..40ed1ab76 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -267,15 +267,18 @@ go_test( sh_test( name = "embeddeddolt_embedded_test", size = "large", - # The script gives the binary the job's -test.timeout=15m per shard; it - # fires (with a goroutine dump) well before Bazel's 1200s action limit - # (eternal under --config=embedded). The job itself also stops at 15m. + # The script gives the binary the job's -test.timeout=20m, as long as + # Bazel's 1200s action limit (eternal under --config=embedded), which + # starts earlier and kills without a goroutine dump (rules_go's wrapper + # ignores the SIGTERM). The trailing -test.timeout=19m overrides it (the + # last flag wins) so Go's own timeout fires first and dumps to test.log. timeout = "eternal", srcs = ["//tools/bazel:go_test_manifest_shard.sh"], args = [ "$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", + "-test.timeout=19m", ], data = [ ":embeddeddolt_test", diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 94216e726..5c4f39246 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -3562,10 +3562,19 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("%s:%s does not contain %q (pr-risk.yml %s):\n%s", c.pkg, c.target, want, c.job, rule) } } + // The script's -test.timeout=20m equals Bazel's 1200s action limit, + // which kills without a goroutine dump; 19m lets Go's fire first. + script := readPolicyFile(t, root, c.script) + if !strings.Contains(script, "-test.timeout=20m") { + t.Errorf("%s no longer passes -test.timeout=20m; revisit %s:%s's -test.timeout=19m", c.script, c.pkg, c.target) + } + if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) { + t.Errorf("%s:%s must pass -test.timeout=19m after the script's own:\n%s", c.pkg, c.target, rule) + } // The script and its manifest must be in the runfiles: without the // manifest the script silently falls back to hash assignment, so // every shard still passes but no longer runs its job's tests. - m := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + m := shardManifestDefault.FindStringSubmatch(script) if m == nil { t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) } @@ -3583,11 +3592,6 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) { t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) } - // The script's -test.timeout=20m equals Bazel's 1200s action limit, - // which kills without a goroutine dump; 19m lets Go's fire first. - if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) { - t.Errorf("cmd/bd:bd_embedded_test must pass -test.timeout=19m after the script's own:\n%s", rule) - } } conformance := risk.job(t, "test-embedded-conformance") diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index e1a89d31d..ac3de4385 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1089,7 +1089,7 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { map[string]string{"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)"}, }, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { - []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)"}, + []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", "-test.timeout=19m"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test": {