* feat(ga-p9iuv.22): add shared tracker proxy storage seam
* fix(ga-p9iuv.22): preserve tracker storage capabilities safely
* test(ga-p9iuv.22): cover tracker adapter label semantics
* test(ga-p9iuv.22): harden tracker adapter contracts
* fix(ga-p9iuv.22): reject invalid proxied tracker stores
* feat(ga-p9iuv.1): route Linear and Jira front doors through proxy store
* test(ga-p9iuv.1): exercise tracker mutations through UOW store
* fix(ga-p9iuv.1): preserve proxied Linear endpoint config
* test(ga-p9iuv.1): cover proxied Linear endpoint and direct config
* feat(ga-p9iuv.19): add typed proxied capability matrix
* feat(proxy): enforce typed capability matrix before provider
* test(proxy): cover command capability rows and claim caps
* test(proxy): check capability rows across topologies
* fix(proxy): validate capabilities before workspace side effects
* fix(proxy): refuse ready claim max rows caps
* feat(proxy): add shared maintenance capability policy
* fix(proxy): key maintenance refusals by command path
* test(proxy): cover nested maintenance refusals
* fix(proxy): enumerate nested maintenance command paths
* fix(proxy): refine compact and nested maintenance policy
* test(proxy): prove maintenance refusals preserve files
* fix(proxy): cover admin and remote maintenance refusals
* fix(proxy): refuse doctor before store initialization
* test(proxy): define history capability command matrix
* fix(proxy): wire history capability refusals to front door
* fix(proxy): refuse top-level sync history path
* fix(proxy): preserve maintenance refusal codes
* feat(proxy): classify history as supported UOW API
* feat(proxy): classify transform capability paths
* test(proxy): harden transform capability matrix
* fix(proxy): drive transform validation from exact rows
* fix(proxy): drive transform matrix from normalized flags
* fix(proxy): bound external backend dials per attempt
* test(proxy): cover bounded blocked backend dial
* fix(proxy): make transform capability policy row-driven
* fix(proxy): make transform capability policy row-driven
* fix(proxy): classify formula swarm and merge-slot refusals
* test(proxy): enforce workflow refusal contract
* fix(proxy): classify Dolt remote history boundaries
* test(proxy): enforce history refusal contract
* test(proxy): cover canceled backend dial cleanup
* test(proxy): cover real TCP blackhole cancellation
* test(proxy): cover external transport lifecycle matrix
* test(proxy): assert stable dial failure diagnostics
* test(proxy): add outage reconnect acceptance matrix
* test(proxy): make unix outage bridge restartable
* test(proxy): bound outage subprocess cleanup
* test(proxy): cover upstream restart with retained endpoint bridge
* test(proxy): stabilize managed-local outage contract
* fix(proxy): retire managed proxy when backend exits
* test(proxy): accept managed child retirement restart
* test(proxy): cover workflow refusals at bd front door
* test(proxy): cover history VC front doors
* test(proxy): pass valid workflow refusal arguments
* test(proxy): assert text and json refusal contracts
* test(proxy): assert exact workflow refusal codes
* test(proxy): guard refusal issue-row stability
* fix(proxy): expose non-mutating capability refusals
* test(proxy): match stable refusal text
* test(proxy): compare refusal artifact snapshots
* test(proxy): assert exact refusal messages
* test(proxy): retain direct workflow behavior
* test(proxy): match nested refusal messages
* test(proxy): cover direct-only front-door refusals
* test(proxy): provide valid direct-only command args
* fix(migrate): repair missing proxied sidecar on reentry
* fix: checkpoint Dolt mode migrations for safe retry
* fix: harden migration journal topology and retries
* fix: fsync migration journal removal
* fix: fail on migration journal directory sync errors
* fix: reject external ownership in every migration phase
* fix: enforce canonical shared migration topology
* fix: enforce live migration topology
* test: add migration front-door and topology regressions
* fix: require physical Dolt roots for migration
* test: strengthen migration front-door contracts
* fix: reject missing or invalid Dolt migration roots
* fix: keep shared migration dry-runs side-effect free
* fix: require Dolt repository identity for migrations
* fix: validate Dolt repository identity metadata
* fix: require typed Dolt repo state identity
* test: expand migration front-door CRUD assertions
* test: make migration JSON dry-run front-door safe
* test: sanitize Dolt environment for migration front door
* test: use cgo binary for migration front door
* test: omit empty Dolt overrides in migration front door
* test: isolate migration front door from test mode
* test: isolate home for migration front door
* test: stop supervised Dolt before reverse migration
* fix: reject empty Dolt head refs
* test: harden migration topology invariants
* test: harden external and shared topology refusals
* test: snapshot migration refusal artifacts
* test: cover external migration refusal phases
* fix: refuse stale proxied sidecars during migration
* test: strengthen migration artifact and external front door checks
* feat: type external migration refusals
* test: assert external refusal starts no proxy
* fix: validate migration journal sidecar topology
* fix: make sidecar recovery phase aware
* fix: honor persisted shared topology during migration
* test: exercise external journal migration front door matrix
* test(tracker): add shared conformance harness
* test: harden external migration matrix fixtures
* test: preserve durable dolt gate lock in matrix
* test: preseed gate lock for refusal matrix
* test: cover malformed migration state front door
* fix: type malformed migration config refusals
* fix: type migration preflight state errors
* test: separate migration JSON stdout from warnings
* test: reach typed malformed journal refusal
* test: assert migration retry phase and idempotence
* test: cover shared migration checkpoint retries
* fix: make reverse migration idempotent without sidecar
* test: cover proxied to shared retries
* fix: derive shared topology from persisted YAML
* fix: validate shared migration no-op state
* fix: allow completed proxied shared no-op
* fix: validate forward proxied no-op sidecar
* test: reject bad already-proxied sidecars
* fix: reject stale proxy controls on no-op
* fix: fail closed on malformed migration workspace yaml
* test: pin migration yaml topology precedence
* Cover forward migration no-op ownership refusals (ga-p9iuv.26.5.2)
Agent-Signature: unknown-model/unknown-reasoning
* Strengthen migration retry sentinel coverage (ga-p9iuv.26.1)
Agent-Signature: unknown-model/unknown-reasoning
* Expand migration fault retry artifact coverage (ga-p9iuv.26.1)
Agent-Signature: unknown-model/unknown-reasoning
* Add reverse migration frontdoor fault matrix (ga-p9iuv.26.1.1)
Agent-Signature: unknown-model/unknown-reasoning
* test: cover shared migration frontdoor fault matrices
* test: strengthen direct migration matrix assertions
* test: preserve migration dependency edges
* test: assert exact migration dependency edges
* test: parse migration dependency edges from batch list
* test: fix shared migration matrix compile
* test: verify shared dependencies before shutdown
* test(proxy): cover history remote front-door topology matrix
* docs: define direct-local ownership handoff contract
* fix(ci): repair pre-review CI failures on proxy parity branch
Five PR-introduced failures across the docs, PR Core, and Proxied Dolt Cmd
lanes. Each fix is scoped to the evidenced defect.
- test/docsync: TestEveryDocsPageIsPublished flagged
docs/architecture/ownership-handoff-contract.md as unpublished. It is an
internal Beads/Gas City handoff contract with no Mintlify frontmatter and
no inbound links, so move it to engdocs/design/ beside its peers rather
than publishing it.
- cmd/bd: TestProxyWorkflowRefusalContractAndNoMutation asserted on
commandDidWrite, a process-wide atomic latch set by 71 non-test sites and
never reset, so it reported an earlier test's write. Baseline and restore
it around the assertion, matching the save/reset/restore idiom already
used in proxied_lookup_failure_test.go and readonly_policy_test.go.
- cmd/bd: the new pre-provider gate short-circuited compact.go's refusal
with new wording, breaking TestProxiedServerCompactDolt/non_dolt_mode_-
rejected in an unchanged test. Restore the long-standing message; the
proxy.compact.unsupported code is unchanged.
- internal/storage/dbproxy/proxy: the unix subtests bound sockets under
t.TempDir(), whose embedded test name overflows macOS's ~104-byte
sun_path limit ("bind: invalid argument"). Add unixSocketPath() to place
them in a short temp dir.
- cmd/bd: migration topology compared recorded and canonical Dolt roots
lexically, so macOS's /var -> /private/var symlink made equivalent paths
compare unequal. Add sameMigrationRoot(), mirroring the existing
resolveExistingPrefix idiom in cmd/bd/doctor/git.go.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
* fix(review): address blocker and major findings from adopt-pr review
Maintainer review fixup for the proxy parity branch, iterations 1-2.
Blockers and majors from the review synthesis and quality scorecard.
Blockers:
- cmd/bd/migrate_dolt_mode.go: resuming from migratePrepared rebuilt the
migration plan from the current argv instead of replaying the recorded
one. A resume that omitted or changed --idle-timeout wrote a sidecar the
target_configured check then rejected, and because that check runs only
after the phase advances past migratePrepared, the block never ran again
— the workspace stayed wedged in proxied-server mode with no in-tool
recovery. Replay j.Sidecar and warn when the supplied value is ignored.
- internal/tracker/store.go: NewStore double-wrapped already-adapted values.
directStore embeds the storage.Storage interface, so *directStore also
satisfies storage.Storage and was re-wrapped as
&directStore{Storage: &directStore{...}}; the inner value can never
satisfy storage.IssueLifecycleStore, which silently degraded
ApplyIssueUpdate to a bare UpdateIssue — no labels, no lifecycle
transaction. Match the tracker's own adapters by concrete type first.
Testing value.(Store) first, as the review suggested, does not work:
storage.Storage is a strict superset of Store, so a raw direct store
satisfies Store too and would be returned un-adapted, relocating the same
bug. The branch order is documented at the call site.
Majors:
- internal/tracker/store.go: directStore.ApplyIssueUpdate now calls
syncIssueLabels, which normalizes both label sets (trim, drop empty) and
restores the semantics the pull path had before this seam moved behind
ApplyIssueUpdate. The un-normalized local copy churned remove+add on
whitespace-only differences and let an empty label reach AddLabel; it is
deleted rather than left as a second, divergent implementation.
- Both regressions were invisible to the suite. Adds
TestNewEngineKeepsDirectStoreAdaptationIntact, TestNewStoreIsIdempotent,
TestDirectStoreApplyIssueUpdateNormalizesLabels, and
TestMigrateToProxiedServer_ResumeReplaysRecordedIdleTimeout (changed /
omitted / never subtests, each also asserting the reverse-migration escape
hatch still recovers). All four were verified to fail against the
pre-fix production code with the predicted symptoms.
Non-gating follow-ups from the scorecard, fixed while nearby:
- cmd/bd/proxy_capability.go: treat ProxyOutcomeNA as "this command has no
such flag" rather than a refusal, so an assert on an N/A row cannot return
a non-nil error carrying an empty message.
- cmd/bd/errors.go: emit the rule's real Mutates value in the JSON refusal
envelope instead of a hardcoded false.
- cmd/bd/main.go: the doctor pre-provider check returned unconditionally,
so a nil result (every doctor subcommand) skipped the legacy-store guard
and autocommit-mode resolution that other skipsStoreInit commands run.
Return only on a real refusal.
- internal/tracker/engine.go: type NewEngine's store parameter as Store so
the seam names its contract, and drop the now-dead applyPullIssueUpdate.
- CHANGELOG.md: record the proxied `bd ready --claim` row-cap refusal, which
was a behavior change shipped without a release note.
Iteration 2 -- gating major (uowStore/directStore external_ref divergence):
- internal/tracker/uow_store.go: GetIssueByExternalRef resolved through a
single merged search over both storage planes, which has no plane
preference: it is a UNION ALL ordered by content and terminating in
`id ASC`, so one external_ref present on both planes resolved to whichever
row's id sorted first. The direct backend is explicitly issues-first --
issueops.GetIssueByExternalRefInTx queries `issues` and falls through to
`wisps` only on sql.ErrNoRows, and says why ("so that pushed ephemeral
beads are found during pull dedup"). The consumer is the pull dedup, which
updates whatever row comes back, so the divergence was a silent write to
the wrong local bead under --proxied-server only. Query the issues plane
first via SkipWisps, then fall back.
The fallback is the merged search rather than the review's literal
Ephemeral=true: that filter contributes an `ephemeral = 1` predicate the
direct backend's `SELECT id FROM wisps WHERE external_ref = ?` does not
have, and wisp-plane rows legitimately carry ephemeral=0 (NoHistory beads;
see types.IssueFilter.EphemeralTier). Taking it literally would have traded
one divergence for a narrower one. Verified: implementing the prescription
fails the ephemeral_flag_clear subtest below.
- internal/tracker/uow_store_engine_test.go: the engine fake now models plane
routing (SkipWisps / Ephemeral / merged) instead of one flat map, so the
contract is expressible at all. Adds
TestUOWStoreGetIssueByExternalRefPrefersIssuePlane, whose wisp id sorts
first so "first merged hit" fails it, and
TestUOWStoreGetIssueByExternalRefFallsBackToWispPlane over both ephemeral
values. Verified to fail against the pre-fix uow_store.go.
- internal/tracker/conformance: the shared suite gains a Wisps plane and
external_ref_resolution_prefers_issue_plane, so this is a contract both
backends are held to rather than one adapter's unit test.
Iteration 2 -- non-gating follow-ups from the scorecard:
- cmd/bd/migrate_dolt_mode.go: scope the migratePrepared sidecar-equivalence
check in runMigrateFromProxiedServer to reverse journals. A forward journal
at that phase records the sidecar it is about to write, so a crash in the
pre-write window leaves none on disk -- the correct state -- and the
unscoped check refused the documented escape hatch by naming a missing
file, stranding the workspace. Tested against TargetMode's forward value so
the guard stays live for both reverse targets, "server" and
"shared-server". Adds
TestMigrateFromProxiedServer_ForwardJournalAtPreparedIsNotSidecarRefused,
verified to fail against the unscoped guard.
- cmd/bd/linear.go: buildLinearClientAPIOnly returned the OAuth client inline
and never reached the WithEndpoint block, so a configured
linear.api_endpoint was honored for an API key and ignored for OAuth. Hoist
the endpoint application after the branch, as Tracker.Init does, and add
the OAuth mirror of the existing endpoint test.
- cmd/bd/ready_max_rows_proxied_test.go: pin the proxied --max-rows refusal
on `bd ready --claim`. It lives in RunE above the dispatch and nothing
below it can enforce a cap, but the existing coverage all sits below it.
Covers both cap sources (--max-rows and BEADS_MAX_ROWS) and the negative
case that the refusal is conditioned on a live cap, not on --claim.
- engdocs/design/ownership-handoff-contract.md: trim the trailing blank line
at EOF flagged by `git diff --check`.
Iteration 3 -- non-gating residual on the iteration-2 fix:
- internal/tracker/uow_store.go: the issues-first fix above split one logical
lookup across two read transactions. uow.RunTxRead opens a fresh unit of
work per call, so calling s.SearchIssues twice left the wisp fallback's
premise -- "the issues plane is already known empty" -- without a shared
snapshot: a concurrent same-ref insert into the issues plane landing between
the two reads could let the merged fallback return the wisp, re-creating the
wrong-row write that ordering exists to prevent. The consumer is the pull
dedup loop, so it runs per imported issue. Both plane reads now share a
single RunTxRead, which is also the shape the direct backend uses (both
planes inside one withReadTx). Plane ordering is unchanged.
- internal/tracker/uow_store_engine_test.go: the engine fake counts the units
of work it hands out, and
TestUOWStoreGetIssueByExternalRefUsesOneReadTransaction pins that count at 1
across the two paths that actually perform both reads -- the wisp fallback
and the miss. An issues-plane hit short-circuits after one read and would
pass either way, so it is not a case. Verified to fail against the
two-transaction version with "opened 2, want 1".
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
524 lines
16 KiB
Go
524 lines
16 KiB
Go
package proxy
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/cenkalti/backoff/v4"
|
|
"golang.org/x/sync/errgroup"
|
|
|
|
"github.com/steveyegge/beads/internal/lockfile"
|
|
"github.com/steveyegge/beads/internal/procid"
|
|
"github.com/steveyegge/beads/internal/storage/dbproxy/identity"
|
|
"github.com/steveyegge/beads/internal/storage/dbproxy/pidfile"
|
|
"github.com/steveyegge/beads/internal/storage/dbproxy/server"
|
|
"github.com/steveyegge/beads/internal/storage/dbproxy/util"
|
|
)
|
|
|
|
const IdleTimeoutNever time.Duration = -1
|
|
|
|
type ProxyOpts struct {
|
|
RootDir string
|
|
Port int
|
|
IdleTimeout time.Duration
|
|
Server server.DatabaseServer
|
|
// StopEpoch is the proxy stop epoch the spawning parent observed under
|
|
// proxy.lock before forking this proxy. ListenAndServe re-reads the
|
|
// epoch immediately before publishing proxy.pid and aborts if it has
|
|
// advanced, so a slow backend start cannot outlast a concurrent
|
|
// `bd dolt stop` and publish a running proxy after the stop returned.
|
|
// Empty means "no stop had ever run", which readStopEpoch also reports
|
|
// for a missing epoch file, so the zero value stays correct for direct
|
|
// (non-forked) callers such as tests.
|
|
StopEpoch string
|
|
// Stats is optional. When non-nil, the proxy records per-event counters
|
|
// against it; tests use Snapshot() to assert. Production code should
|
|
// leave this nil.
|
|
Stats *Stats
|
|
}
|
|
|
|
type proxyServer struct {
|
|
rootDir string
|
|
port int
|
|
idleTimeout time.Duration
|
|
server server.DatabaseServer
|
|
stats *Stats
|
|
stopEpoch string
|
|
|
|
logger *log.Logger
|
|
listener net.Listener
|
|
activeConns atomic.Int64
|
|
conns errgroup.Group
|
|
}
|
|
|
|
const (
|
|
PIDFileName = "proxy.pid"
|
|
LogFileName = "proxy.log"
|
|
LockFileName = "proxy.lock"
|
|
)
|
|
|
|
// LockHeldExitCode is the exit code a child proxy should use when
|
|
// ListenAndServe returns ErrLockHeld. The spawning parent treats this
|
|
// (EX_TEMPFAIL) as "lost the spawn race" and retries via readAndDial.
|
|
const LockHeldExitCode = 75
|
|
|
|
// ErrLockHeld is returned from ListenAndServe when another proxy already
|
|
// holds proxy.lock for the same rootDir. It is a normal "lost the race"
|
|
// outcome, not a failure: callers spawned as children should map it to
|
|
// LockHeldExitCode and exit cleanly.
|
|
var ErrLockHeld = errors.New("proxy lock held by another proxy on this rootDir")
|
|
|
|
const (
|
|
serverReadyTimeout = 30 * time.Second
|
|
readyDialTimeout = 2 * time.Second
|
|
readyInitialBackoff = 50 * time.Millisecond
|
|
readyMaxBackoff = 1 * time.Second
|
|
backendDialTimeout = 5 * time.Second
|
|
idleWatcherMinInterval = 1 * time.Second
|
|
backendStopTimeout = 5 * time.Minute
|
|
tcpKeepAlivePeriod = 30 * time.Second
|
|
backendHealthInterval = 100 * time.Millisecond
|
|
)
|
|
|
|
var errIdleTimeout = errors.New("idle timeout reached")
|
|
|
|
func NewProxyServer(opts ProxyOpts) *proxyServer {
|
|
return &proxyServer{
|
|
rootDir: opts.RootDir,
|
|
port: opts.Port,
|
|
idleTimeout: opts.IdleTimeout,
|
|
server: opts.Server,
|
|
stats: opts.Stats,
|
|
stopEpoch: opts.StopEpoch,
|
|
}
|
|
}
|
|
|
|
func (p *proxyServer) tracef(format string, args ...any) {
|
|
p.logger.Printf(format, args...)
|
|
}
|
|
|
|
func (p *proxyServer) ListenAndServe(parentCtx context.Context) error {
|
|
lock, err := util.TryLock(filepath.Join(p.rootDir, LockFileName))
|
|
if err != nil {
|
|
if lockfile.IsLocked(err) {
|
|
return ErrLockHeld
|
|
}
|
|
return fmt.Errorf("acquire %s: %w", LockFileName, err)
|
|
}
|
|
// proxy.lock is held for the proxy's whole lifetime, but a doomed start
|
|
// must be able to release it early (before its backend teardown) without
|
|
// the deferred release double-unlocking. Only the main goroutine touches
|
|
// this.
|
|
lockHeld := true
|
|
releaseLock := func() {
|
|
if lockHeld {
|
|
lockHeld = false
|
|
lock.Unlock()
|
|
}
|
|
}
|
|
defer releaseLock()
|
|
if err := clearSpawnMarkerAfterLock(p.rootDir); err != nil {
|
|
return fmt.Errorf("clear proxy spawn marker: %w", err)
|
|
}
|
|
|
|
// Fast-abort: a concurrent `bd dolt stop` advances the stop epoch before
|
|
// waiting (briefly) for proxy.lock, so an epoch that moved between the
|
|
// spawning parent's read and this child taking the lock dooms this start.
|
|
// Abort before opening any listener or booting the backend: the stopper
|
|
// then observes a free lock within milliseconds instead of after a full
|
|
// doomed boot-and-teardown cycle.
|
|
if changed, err := stopEpochChanged(p.rootDir, p.stopEpoch); err != nil {
|
|
return fmt.Errorf("check proxy stop epoch after acquiring %s: %w", LockFileName, err)
|
|
} else if changed {
|
|
return fmt.Errorf("%w for %s: stop epoch advanced before startup", errStartInterrupted, p.rootDir)
|
|
}
|
|
|
|
logPath := filepath.Join(p.rootDir, LogFileName)
|
|
f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) // #nosec G304 -- logPath is derived from operator-supplied config, not untrusted request input
|
|
if err != nil {
|
|
return fmt.Errorf("open proxy log %q: %w", logPath, err)
|
|
}
|
|
p.logger = log.New(f, "[proxy] ", log.LstdFlags|log.Lmicroseconds)
|
|
defer func() { _ = f.Close() }()
|
|
|
|
ctx, cancel := context.WithCancel(parentCtx)
|
|
defer cancel()
|
|
|
|
// Install signal handlers BEFORE Listen. Without this, Go's default
|
|
// SIGTERM action terminates the process during the startup window
|
|
// (Listen, pidfile write, backend Start, readiness wait), bypassing all
|
|
// deferred cleanup including RemoveDatabaseProxyPidFile.
|
|
sigCh := make(chan os.Signal, 1)
|
|
signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
|
|
defer signal.Stop(sigCh)
|
|
|
|
var sigReceived atomic.Bool
|
|
go func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-sigCh:
|
|
sigReceived.Store(true)
|
|
p.stats.IncSignalReceived()
|
|
cancel()
|
|
}
|
|
}()
|
|
|
|
// Watch the stop epoch across the whole startup window (listen, backend
|
|
// Start, readiness wait). A stop that begins mid-boot advances the epoch
|
|
// first and then waits only shutdownConfirmDeadline for proxy.lock, which
|
|
// this child holds throughout the boot; without a watcher the child would
|
|
// notice the doomed start only at the pre-publish fence, potentially tens
|
|
// of seconds later. Canceling ctx aborts the backend Start / ready wait
|
|
// within about one poll interval. Transient epoch read errors are ignored
|
|
// here; the pre-publish fence remains the authoritative check.
|
|
epochWatchCtx, epochWatchCancel := context.WithCancel(context.Background())
|
|
epochWatchDone := make(chan struct{})
|
|
go func() {
|
|
defer close(epochWatchDone)
|
|
ticker := time.NewTicker(openPollInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-epochWatchCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
changed, err := stopEpochChanged(p.rootDir, p.stopEpoch)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if changed {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
stopEpochWatch := func() {
|
|
epochWatchCancel()
|
|
<-epochWatchDone
|
|
}
|
|
defer stopEpochWatch()
|
|
|
|
// abortInterruptedStart tears down a doomed start whose stop epoch
|
|
// advanced mid-boot. The interrupting stop is polling proxy.lock under a
|
|
// budget (shutdownConfirmDeadline) far smaller than a backend stop can
|
|
// take, and nothing has been published, so release the lock BEFORE the
|
|
// backend teardown instead of starving the stopper into its timeout.
|
|
abortInterruptedStart := func() error {
|
|
p.stats.IncBackendStop()
|
|
releaseLock()
|
|
_ = stopBackendBounded(p.server)
|
|
return fmt.Errorf("%w for %s: stop epoch advanced during startup", errStartInterrupted, p.rootDir)
|
|
}
|
|
|
|
addr := fmt.Sprintf("127.0.0.1:%d", p.port)
|
|
|
|
ln, err := net.Listen("tcp", addr)
|
|
if err != nil {
|
|
return fmt.Errorf("listen on %s: %w", addr, err)
|
|
}
|
|
|
|
p.listener = ln
|
|
defer func() { _ = ln.Close() }()
|
|
p.stats.IncListenAndServe()
|
|
dataPort, ok := ln.Addr().(*net.TCPAddr)
|
|
if !ok {
|
|
return fmt.Errorf("proxy: unexpected data listener address %T", ln.Addr())
|
|
}
|
|
|
|
if _, err := identity.WriteSecret(p.rootDir); err != nil {
|
|
return fmt.Errorf("write proxy secret: %w", err)
|
|
}
|
|
|
|
var identMu sync.RWMutex
|
|
identReply := identity.IdentReply{
|
|
Schema: pidfile.SchemaV2,
|
|
Role: pidfile.KindProxy,
|
|
DataPort: dataPort.Port,
|
|
}
|
|
control, err := startControl(p.rootDir, func() identity.IdentReply {
|
|
identMu.RLock()
|
|
defer identMu.RUnlock()
|
|
return identReply
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("start control listener: %w", err)
|
|
}
|
|
defer func() { _ = control.Close() }()
|
|
|
|
p.stats.IncBackendStart()
|
|
if err := p.server.Start(ctx); err != nil {
|
|
// Start failed with no backend left running (Start cleans up its own
|
|
// failure), so there is no teardown to move off the lock; classifying
|
|
// the epoch-watcher cancellation just keeps the child's exit reason
|
|
// precise for the spawning parent.
|
|
if changed, cerr := stopEpochChanged(p.rootDir, p.stopEpoch); cerr == nil && changed {
|
|
return fmt.Errorf("%w for %s: stop epoch advanced during backend start (%v)", errStartInterrupted, p.rootDir, err)
|
|
}
|
|
return fmt.Errorf("start database server: %w", err)
|
|
}
|
|
|
|
if err := waitForServerReady(ctx, p.server, serverReadyTimeout); err != nil {
|
|
if changed, cerr := stopEpochChanged(p.rootDir, p.stopEpoch); cerr == nil && changed {
|
|
return abortInterruptedStart()
|
|
}
|
|
p.stats.IncBackendStop()
|
|
_ = stopBackendBounded(p.server)
|
|
return fmt.Errorf("database server not ready: %w", err)
|
|
}
|
|
birth, err := procid.Capture(os.Getpid())
|
|
if err != nil {
|
|
p.stats.IncBackendStop()
|
|
_ = stopBackendBounded(p.server)
|
|
return fmt.Errorf("capture proxy birth identity: %w", err)
|
|
}
|
|
rootID, err := identity.RootID(p.rootDir)
|
|
if err != nil {
|
|
p.stats.IncBackendStop()
|
|
_ = stopBackendBounded(p.server)
|
|
return fmt.Errorf("resolve proxy root identity: %w", err)
|
|
}
|
|
upstreamID := p.server.ID(ctx)
|
|
identMu.Lock()
|
|
identReply.RootID = rootID
|
|
identReply.UpstreamID = upstreamID
|
|
identReply.PID = os.Getpid()
|
|
identReply.Birth = string(birth)
|
|
identReply.ControlPort = control.Port()
|
|
identMu.Unlock()
|
|
|
|
// Last fence before publishing: the spawn marker was cleared when this
|
|
// process took proxy.lock, so a `bd dolt stop` that began during a slow
|
|
// backend start has no record of this attempt. It did advance the stop
|
|
// epoch first, so re-check it here and abort instead of publishing a
|
|
// running proxy after that stop returned. The startup epoch watcher is
|
|
// stopped (synchronously) first: past this fence a stop finds the
|
|
// published proxy.pid and stops the proxy through it, so a watcher
|
|
// cancellation must not race the publish.
|
|
stopEpochWatch()
|
|
if changed, err := stopEpochChanged(p.rootDir, p.stopEpoch); err != nil {
|
|
p.stats.IncBackendStop()
|
|
_ = stopBackendBounded(p.server)
|
|
return fmt.Errorf("re-check proxy stop epoch before publish: %w", err)
|
|
} else if changed {
|
|
return abortInterruptedStart()
|
|
}
|
|
|
|
if err := pidfile.Write(p.rootDir, PIDFileName, pidfile.PidFile{
|
|
Pid: os.Getpid(),
|
|
Port: dataPort.Port,
|
|
UpstreamID: upstreamID,
|
|
Schema: pidfile.SchemaV2,
|
|
Kind: pidfile.KindProxy,
|
|
Birth: string(birth),
|
|
RootID: rootID,
|
|
ControlPort: control.Port(),
|
|
}); err != nil {
|
|
p.stats.IncBackendStop()
|
|
_ = stopBackendBounded(p.server)
|
|
return fmt.Errorf("write pid file: %w", err)
|
|
}
|
|
defer func() { _ = pidfile.Remove(p.rootDir, PIDFileName) }()
|
|
|
|
g, gctx := errgroup.WithContext(ctx)
|
|
g.Go(func() error {
|
|
<-gctx.Done()
|
|
_ = p.listener.Close()
|
|
_ = control.Close()
|
|
return nil
|
|
})
|
|
g.Go(func() error { return p.idleWatcher(gctx) })
|
|
g.Go(func() error {
|
|
t := time.NewTicker(backendHealthInterval)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-gctx.Done():
|
|
return nil
|
|
case <-t.C:
|
|
if !p.server.Running(gctx) {
|
|
return errors.New("database server exited")
|
|
}
|
|
}
|
|
}
|
|
})
|
|
g.Go(func() error { return p.acceptLoop(gctx) })
|
|
|
|
runErr := g.Wait()
|
|
_ = p.conns.Wait()
|
|
p.stats.IncBackendStop()
|
|
stopErr := stopBackendBounded(p.server)
|
|
if stopErr != nil {
|
|
stopErr = fmt.Errorf("stop database server: %w", stopErr)
|
|
}
|
|
if errors.Is(runErr, errIdleTimeout) || sigReceived.Load() {
|
|
runErr = nil
|
|
}
|
|
return errors.Join(runErr, stopErr)
|
|
}
|
|
|
|
func stopBackendBounded(s server.DatabaseServer) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), backendStopTimeout)
|
|
defer cancel()
|
|
return s.Stop(ctx)
|
|
}
|
|
|
|
func (p *proxyServer) idleWatcher(ctx context.Context) error {
|
|
if p.idleTimeout <= 0 {
|
|
<-ctx.Done()
|
|
return nil
|
|
}
|
|
interval := p.idleTimeout / 4
|
|
if interval < idleWatcherMinInterval {
|
|
interval = idleWatcherMinInterval
|
|
}
|
|
p.tracef("idleWatcher start (timeout=%s, tick=%s)", p.idleTimeout, interval)
|
|
tick := time.NewTicker(interval)
|
|
defer tick.Stop()
|
|
var idleSince time.Time
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
p.tracef("idleWatcher exit (ctx done)")
|
|
return nil
|
|
case <-tick.C:
|
|
if n := p.activeConns.Load(); n > 0 {
|
|
if !idleSince.IsZero() {
|
|
p.tracef("idleWatcher cleared (active=%d)", n)
|
|
idleSince = time.Time{}
|
|
}
|
|
continue
|
|
}
|
|
if idleSince.IsZero() {
|
|
p.tracef("idleWatcher armed")
|
|
idleSince = time.Now()
|
|
continue
|
|
}
|
|
if time.Since(idleSince) >= p.idleTimeout {
|
|
p.tracef("idleWatcher expired after %s, shutting down", p.idleTimeout)
|
|
p.stats.IncIdleTimeout()
|
|
return errIdleTimeout
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *proxyServer) acceptLoop(ctx context.Context) error {
|
|
p.tracef("acceptLoop start (addr=%s)", p.listener.Addr())
|
|
for {
|
|
conn, err := p.listener.Accept()
|
|
if err != nil {
|
|
if errors.Is(err, net.ErrClosed) || ctx.Err() != nil {
|
|
p.tracef("acceptLoop exit (ctx=%v)", ctx.Err())
|
|
return nil
|
|
}
|
|
// Surface non-shutdown accept errors to the errgroup so the
|
|
// proxy fails fast instead of busy-looping. Specific errors that
|
|
// warrant retry (e.g. transient EMFILE under load) can be added
|
|
// here as the need arises.
|
|
p.tracef("acceptLoop error: %v", err)
|
|
p.stats.IncAcceptError()
|
|
return fmt.Errorf("accept: %w", err)
|
|
}
|
|
if tc, ok := conn.(*net.TCPConn); ok {
|
|
_ = tc.SetKeepAlive(true)
|
|
_ = tc.SetKeepAlivePeriod(tcpKeepAlivePeriod)
|
|
}
|
|
p.tracef("acceptLoop accepted (remote=%s)", conn.RemoteAddr())
|
|
p.stats.IncAccept()
|
|
p.conns.Go(func() error {
|
|
return p.handleConn(ctx, conn)
|
|
})
|
|
}
|
|
}
|
|
|
|
func (p *proxyServer) handleConn(ctx context.Context, client net.Conn) error {
|
|
addr := client.RemoteAddr()
|
|
p.tracef("handleConn(%s) start", addr)
|
|
p.activeConns.Add(1)
|
|
defer func() {
|
|
p.activeConns.Add(-1)
|
|
p.tracef("handleConn(%s) end (active=%d)", addr, p.activeConns.Load())
|
|
}()
|
|
|
|
p.stats.IncBackendDialAttempt()
|
|
dialCtx, cancelDial := context.WithTimeout(ctx, backendDialTimeout)
|
|
backend, err := p.server.Dial(dialCtx)
|
|
cancelDial()
|
|
if err != nil {
|
|
p.tracef("handleConn(%s) backend dial error: %v", addr, err)
|
|
p.stats.IncBackendDialError()
|
|
_ = client.Close()
|
|
return err
|
|
}
|
|
p.tracef("handleConn(%s) backend dial ok", addr)
|
|
p.stats.IncBackendDialSuccess()
|
|
p.stats.IncHandledConn()
|
|
|
|
done := make(chan struct{})
|
|
var doneOnce sync.Once
|
|
finish := func() { doneOnce.Do(func() { close(done) }) }
|
|
|
|
var g errgroup.Group
|
|
g.Go(func() error {
|
|
select {
|
|
case <-ctx.Done():
|
|
p.tracef("handleConn(%s) ctx canceled, force-closing", addr)
|
|
_ = client.Close()
|
|
_ = backend.Close()
|
|
case <-done:
|
|
}
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
defer finish()
|
|
defer func() { _ = backend.Close() }()
|
|
defer func() { _ = client.Close() }()
|
|
n, err := io.Copy(backend, client)
|
|
p.stats.AddBytesClientToBackend(n)
|
|
p.tracef("handleConn(%s) client→backend done (n=%d, err=%v)", addr, n, err)
|
|
return err
|
|
})
|
|
g.Go(func() error {
|
|
defer finish()
|
|
defer func() { _ = backend.Close() }()
|
|
defer func() { _ = client.Close() }()
|
|
n, err := io.Copy(client, backend)
|
|
p.stats.AddBytesBackendToClient(n)
|
|
p.tracef("handleConn(%s) backend→client done (n=%d, err=%v)", addr, n, err)
|
|
return err
|
|
})
|
|
return g.Wait()
|
|
}
|
|
|
|
func waitForServerReady(ctx context.Context, s server.DatabaseServer, timeout time.Duration) error {
|
|
bo := backoff.NewExponentialBackOff()
|
|
bo.InitialInterval = readyInitialBackoff
|
|
bo.MaxInterval = readyMaxBackoff
|
|
bo.MaxElapsedTime = timeout
|
|
|
|
return backoff.Retry(func() error {
|
|
if !s.Running(ctx) {
|
|
return errors.New("database server not running")
|
|
}
|
|
dialCtx, cancel := context.WithTimeout(ctx, readyDialTimeout)
|
|
defer cancel()
|
|
conn, err := s.Dial(dialCtx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_ = conn.Close()
|
|
return nil
|
|
}, backoff.WithContext(bo, ctx))
|
|
}
|