beads/internal/storage/sqlcount/sqlcount.go
Gas City Adopt-PR fe54ac09ef feat(httpapi,issueops): raise batchApply envelope to 1000 items with bounded large-apply handling
POST /v0/beads/issues:batchApply now accepts up to 1000 items (up from 100);
issueops.MaxApplyBatchItems moves accordingly and the request body cap rises
to 16 MiB. The whole plan still applies in exactly ONE transaction. A request
over 100 items runs under a separate, flat, operator-configured whole-run
budget (`bd serve --large-apply-ceiling`, 5 minutes by default) instead of
the ordinary 60s request deadline; a request at or under 100 items is
unaffected and keeps exactly the 60s deadline and 15s commit-retry ceiling it
always had.

Concurrency and shutdown are bounded, not just documented:
- Oversized requests serialize through a dedicated one-wide semaphore, and at
  most one caller may wait for that semaphore at a time (a CAS-guarded single
  waiter) — a burst of oversized requests is shed with 503 rather than
  queuing and starving ordinary traffic behind the general semaphore.
- The run deadline is stamped only once the slot is actually acquired, not
  at the time the request arrived.
- Graceful shutdown closes new admissions before computing its drain budget,
  then waits out an in-flight large apply up to its own remaining deadline
  (plus a few seconds' grace for the response to write); any large apply
  still queued for the semaphore when shutdown begins is refused rather than
  extending the drain further. A queued caller that disconnects gets a clean
  503, never a misclassified context error. `bd serve`'s orchestrator stop
  grace must be at least `--large-apply-ceiling` — see
  engdocs/SERVE_RUNBOOK.md.
- ApplyBatch's commit-retry budget only extends past the ordinary 15s ceiling
  on this explicit large-apply path (via a context marker), with headroom
  reserved so the final retry attempt can't be killed mid-commit by the
  context's own expiry.

A new `issues.batchApplyLarge` capability token on `GET /v0/beads/context`
lets a client learn whether a given server accepts the raised envelope.

Wall-clock measurement (embedded Dolt, whole-transaction build+commit) backs
the 5-minute ceiling default: 356 items ~12.6s (23.8x headroom), 1000 items
~57.5s (5.2x headroom).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-01 12:37:40 +00:00

417 lines
14 KiB
Go

// Package sqlcount wraps a database/sql/driver.Connector so a test or
// benchmark can count how many SQL statements a code path actually issues
// against a real driver, and optionally inject artificial per-statement
// latency to approximate a database that is not co-located with the caller.
//
// It exists for the large-batch-apply measurement work: issueops.ApplyBatchInTx
// takes a concrete *sql.Tx, not an interface, so the only seam available to
// count its real round trips is the database/sql/driver boundary itself. Wrapping the
// driver.Connector a backend already builds — mysql.NewConnector for the Dolt
// server backend, doltembed.NewConnector for the embedded backend — is the
// one construction point both backends share, so one wrapper measures both
// without touching either driver or any production code path.
package sqlcount
import (
"context"
"database/sql/driver"
"fmt"
"sync/atomic"
"time"
)
// Counts tracks how many times each driver entry point that represents one
// SQL statement or round trip fired. The zero value is ready to use. Safe
// for concurrent use.
type Counts struct {
prepare int64
exec int64
query int64
stmtExec int64
stmtQuery int64
begin int64
commit int64
rollback int64
}
// Snapshot is a point-in-time, plain copy of Counts, safe to diff, log or
// assert on without synchronization.
type Snapshot struct {
Prepare int64
Exec int64
Query int64
StmtExec int64
StmtQuery int64
Begin int64
Commit int64
Rollback int64
}
// Total is every round trip this wrapper attributes to an executed SQL
// statement: a Prepare, a driver-level Exec/Query issued without a separate
// prepare, or an Exec/Query against an already-prepared statement. Begin,
// Commit and Rollback are reported on Snapshot but excluded here, because a
// caller measuring one transaction's BODY (issueops.ApplyBatchInTx, for
// instance) opens and closes that transaction outside the measured window.
func (s Snapshot) Total() int64 {
return s.Prepare + s.Exec + s.Query + s.StmtExec + s.StmtQuery
}
// Sub returns the per-field difference s-prior, for isolating one call's
// contribution on a longer-lived connection.
func (s Snapshot) Sub(prior Snapshot) Snapshot {
return Snapshot{
Prepare: s.Prepare - prior.Prepare,
Exec: s.Exec - prior.Exec,
Query: s.Query - prior.Query,
StmtExec: s.StmtExec - prior.StmtExec,
StmtQuery: s.StmtQuery - prior.StmtQuery,
Begin: s.Begin - prior.Begin,
Commit: s.Commit - prior.Commit,
Rollback: s.Rollback - prior.Rollback,
}
}
// Snapshot reads every counter. The read is not atomic ACROSS fields — a
// concurrent writer could interleave between two loads — which matches every
// other multi-atomic snapshot in this tree taken for one log line or report.
func (c *Counts) Snapshot() Snapshot {
return Snapshot{
Prepare: atomic.LoadInt64(&c.prepare),
Exec: atomic.LoadInt64(&c.exec),
Query: atomic.LoadInt64(&c.query),
StmtExec: atomic.LoadInt64(&c.stmtExec),
StmtQuery: atomic.LoadInt64(&c.stmtQuery),
Begin: atomic.LoadInt64(&c.begin),
Commit: atomic.LoadInt64(&c.commit),
Rollback: atomic.LoadInt64(&c.rollback),
}
}
// Reset zeroes every counter, for reuse across benchmark iterations.
func (c *Counts) Reset() {
atomic.StoreInt64(&c.prepare, 0)
atomic.StoreInt64(&c.exec, 0)
atomic.StoreInt64(&c.query, 0)
atomic.StoreInt64(&c.stmtExec, 0)
atomic.StoreInt64(&c.stmtQuery, 0)
atomic.StoreInt64(&c.begin, 0)
atomic.StoreInt64(&c.commit, 0)
atomic.StoreInt64(&c.rollback, 0)
}
// Total is c.Snapshot().Total(), for a caller that wants one number without
// composing the two calls.
func (c *Counts) Total() int64 { return c.Snapshot().Total() }
// options carries WrapConnector's optional behavior.
type options struct {
latency time.Duration
}
// Option configures WrapConnector.
type Option func(*options)
// WithLatency injects a sleep before every counted round trip — Prepare,
// Exec, Query, a prepared statement's Exec/Query, Begin, Commit and Rollback
// alike — so a co-located backend can approximate the wall-clock cost of a
// database that is NOT co-located with the caller. It respects the caller's
// context: a canceled or expired context aborts the sleep and returns the
// context's error rather than padding out a call that is already doomed.
func WithLatency(d time.Duration) Option {
return func(o *options) { o.latency = d }
}
// WrapConnector wraps inner so every statement-shaped call on the
// connections it hands out increments counts, and optionally sleeps first.
// inner is the driver.Connector a backend already constructs
// (mysql.NewConnector for the Dolt server backend, doltembed.NewConnector
// for the embedded backend); wrapping it rather than registering a new
// driver under a new name is what lets a test open a SECOND, counted
// connection against the SAME already-migrated embedded directory or the
// SAME live Dolt server a backend's own store already opened, so the
// production open/migrate path never has to know this wrapper exists.
func WrapConnector(inner driver.Connector, counts *Counts, opts ...Option) driver.Connector {
o := &options{}
for _, opt := range opts {
opt(o)
}
return &wrappedConnector{inner: inner, counts: counts, opts: o}
}
type wrappedConnector struct {
inner driver.Connector
counts *Counts
opts *options
}
func (c *wrappedConnector) Connect(ctx context.Context) (driver.Conn, error) {
inner, err := c.inner.Connect(ctx)
if err != nil {
return nil, err
}
return &wrappedConn{inner: inner, counts: c.counts, opts: c.opts}, nil
}
func (c *wrappedConnector) Driver() driver.Driver {
return &wrappedDriver{inner: c.inner.Driver(), counts: c.counts, opts: c.opts}
}
// wrappedDriver exists only to satisfy driver.Connector.Driver's return
// type. database/sql calls Driver() for error classification and
// sql.DB.Driver(); this wrapper never uses it to open a second connection
// outside wrappedConnector.Connect.
type wrappedDriver struct {
inner driver.Driver
counts *Counts
opts *options
}
func (d *wrappedDriver) Open(name string) (driver.Conn, error) {
inner, err := d.inner.Open(name)
if err != nil {
return nil, err
}
return &wrappedConn{inner: inner, counts: d.counts, opts: d.opts}, nil
}
// sleep injects the configured latency, honoring ctx cancellation.
func (o *options) sleep(ctx context.Context) error {
if o.latency <= 0 {
return nil
}
t := time.NewTimer(o.latency)
defer t.Stop()
select {
case <-t.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// wrappedConn wraps a driver.Conn. It implements every optional modern
// interface (ExecerContext, QueryerContext, ConnPrepareContext,
// ConnBeginTx, Pinger, SessionResetter, Validator, NamedValueChecker),
// delegating to the inner connection's implementation when present and
// falling back to the legacy path — or driver.ErrSkip — when not, so
// database/sql's behavior with the wrapper in place matches its behavior
// with the inner driver alone.
//
// NamedValueChecker is always implemented (never omitted) because
// database/sql decides parameter-conversion behavior by checking interface
// satisfaction on the WRAPPER's concrete type, not the inner driver's:
// omitting it here would silently change conversion behavior relative to
// running against the inner driver directly, regardless of what the inner
// driver itself implements.
type wrappedConn struct {
inner driver.Conn
counts *Counts
opts *options
}
func (c *wrappedConn) Prepare(query string) (driver.Stmt, error) {
atomic.AddInt64(&c.counts.prepare, 1)
inner, err := c.inner.Prepare(query)
if err != nil {
return nil, err
}
return &wrappedStmt{inner: inner, counts: c.counts, opts: c.opts}, nil
}
func (c *wrappedConn) Close() error { return c.inner.Close() }
//nolint:staticcheck // legacy driver.Conn.Begin, required by the interface; BeginTx below is preferred when available.
func (c *wrappedConn) Begin() (driver.Tx, error) {
atomic.AddInt64(&c.counts.begin, 1)
inner, err := c.inner.Begin() //nolint:staticcheck
if err != nil {
return nil, err
}
return &wrappedTx{inner: inner, counts: c.counts}, nil
}
func (c *wrappedConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
pc, ok := c.inner.(driver.ConnPrepareContext)
if !ok {
return c.Prepare(query)
}
if err := c.opts.sleep(ctx); err != nil {
return nil, err
}
atomic.AddInt64(&c.counts.prepare, 1)
inner, err := pc.PrepareContext(ctx, query)
if err != nil {
return nil, err
}
return &wrappedStmt{inner: inner, counts: c.counts, opts: c.opts}, nil
}
func (c *wrappedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
execer, ok := c.inner.(driver.ExecerContext)
if !ok {
return nil, driver.ErrSkip
}
if err := c.opts.sleep(ctx); err != nil {
return nil, err
}
atomic.AddInt64(&c.counts.exec, 1)
return execer.ExecContext(ctx, query, args)
}
func (c *wrappedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
queryer, ok := c.inner.(driver.QueryerContext)
if !ok {
return nil, driver.ErrSkip
}
if err := c.opts.sleep(ctx); err != nil {
return nil, err
}
atomic.AddInt64(&c.counts.query, 1)
return queryer.QueryContext(ctx, query, args)
}
func (c *wrappedConn) BeginTx(ctx context.Context, txOpts driver.TxOptions) (driver.Tx, error) {
bt, ok := c.inner.(driver.ConnBeginTx)
if !ok {
return c.Begin()
}
if err := c.opts.sleep(ctx); err != nil {
return nil, err
}
atomic.AddInt64(&c.counts.begin, 1)
inner, err := bt.BeginTx(ctx, txOpts)
if err != nil {
return nil, err
}
return &wrappedTx{inner: inner, counts: c.counts}, nil
}
func (c *wrappedConn) Ping(ctx context.Context) error {
if p, ok := c.inner.(driver.Pinger); ok {
return p.Ping(ctx)
}
return nil
}
func (c *wrappedConn) ResetSession(ctx context.Context) error {
if r, ok := c.inner.(driver.SessionResetter); ok {
return r.ResetSession(ctx)
}
return nil
}
func (c *wrappedConn) IsValid() bool {
if v, ok := c.inner.(driver.Validator); ok {
return v.IsValid()
}
return true
}
func (c *wrappedConn) CheckNamedValue(nv *driver.NamedValue) error {
if chk, ok := c.inner.(driver.NamedValueChecker); ok {
return chk.CheckNamedValue(nv)
}
return driver.ErrSkip
}
// wrappedStmt wraps a driver.Stmt the same way wrappedConn wraps a
// driver.Conn: delegate to a modern optional interface when the inner
// statement implements it, else fall back to the legacy path, and always
// implement NamedValueChecker for the reason documented on wrappedConn.
type wrappedStmt struct {
inner driver.Stmt
counts *Counts
opts *options
}
func (s *wrappedStmt) Close() error { return s.inner.Close() }
func (s *wrappedStmt) NumInput() int { return s.inner.NumInput() }
//nolint:staticcheck // legacy driver.Stmt.Exec, required by the interface and used as the ExecContext fallback below.
func (s *wrappedStmt) Exec(args []driver.Value) (driver.Result, error) {
atomic.AddInt64(&s.counts.stmtExec, 1)
return s.inner.Exec(args) //nolint:staticcheck
}
//nolint:staticcheck // legacy driver.Stmt.Query, required by the interface and used as the QueryContext fallback below.
func (s *wrappedStmt) Query(args []driver.Value) (driver.Rows, error) {
atomic.AddInt64(&s.counts.stmtQuery, 1)
return s.inner.Query(args) //nolint:staticcheck
}
func (s *wrappedStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
if err := s.opts.sleep(ctx); err != nil {
return nil, err
}
if se, ok := s.inner.(driver.StmtExecContext); ok {
atomic.AddInt64(&s.counts.stmtExec, 1)
return se.ExecContext(ctx, args)
}
dargs, err := namedToOrdinal(args)
if err != nil {
return nil, err
}
atomic.AddInt64(&s.counts.stmtExec, 1)
return s.inner.Exec(dargs) //nolint:staticcheck
}
func (s *wrappedStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
if err := s.opts.sleep(ctx); err != nil {
return nil, err
}
if sq, ok := s.inner.(driver.StmtQueryContext); ok {
atomic.AddInt64(&s.counts.stmtQuery, 1)
return sq.QueryContext(ctx, args)
}
dargs, err := namedToOrdinal(args)
if err != nil {
return nil, err
}
atomic.AddInt64(&s.counts.stmtQuery, 1)
return s.inner.Query(dargs) //nolint:staticcheck
}
func (s *wrappedStmt) CheckNamedValue(nv *driver.NamedValue) error {
if chk, ok := s.inner.(driver.NamedValueChecker); ok {
return chk.CheckNamedValue(nv)
}
return driver.ErrSkip
}
// namedToOrdinal converts modern named-value args back to the legacy
// ordinal-positioned driver.Value slice Exec/Query take, for an inner
// statement that implements neither StmtExecContext nor StmtQueryContext.
// Every driver this wrapper targets (mysql, dolthub/driver) implements the
// modern interfaces, so this path is exercised only as a defensive
// fallback, never in the measured code path.
func namedToOrdinal(args []driver.NamedValue) ([]driver.Value, error) {
out := make([]driver.Value, len(args))
for _, a := range args {
if a.Name != "" {
return nil, fmt.Errorf("sqlcount: named parameter %q not supported by the wrapped driver's legacy Exec/Query", a.Name)
}
if a.Ordinal < 1 || a.Ordinal > len(args) {
return nil, fmt.Errorf("sqlcount: parameter ordinal %d out of range for %d args", a.Ordinal, len(args))
}
out[a.Ordinal-1] = a.Value
}
return out, nil
}
// wrappedTx wraps a driver.Tx purely to count Commit/Rollback; it does not
// change the transaction's behavior.
type wrappedTx struct {
inner driver.Tx
counts *Counts
}
func (t *wrappedTx) Commit() error {
atomic.AddInt64(&t.counts.commit, 1)
return t.inner.Commit()
}
func (t *wrappedTx) Rollback() error {
atomic.AddInt64(&t.counts.rollback, 1)
return t.inner.Rollback()
}