diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 4fe8b38c2..59ca95906 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -28,6 +28,12 @@ # incremental --write keeps pinned lines where they are; a pin is a # reviewed choice, not a regeneration of this block. # +# TestProxiedServerOutageReconnectAcceptanceMatrix is pinned to shard 1 +# (the lightest in recent Main runs, ~6.5 min) instead of hashing onto +# shard 14, which already runs ~13.5 min of the 15m timeout. These jobs +# have no socat, so today it skips here; the pin keeps shard 14 safe if +# socat is ever installed. +# # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator instead. Known blind spots # of the cost proxy (it counts inits, not the work done per init): @@ -51,6 +57,7 @@ 15 1 TestProxiedServerDepConcurrent 15 1 TestProxiedServerListSortRetiresTheWalk 15 1 TestProxiedServerMolLastActivity +15 1 TestProxiedServerOutageReconnectAcceptanceMatrix 15 1 TestProxiedServerReopenConcurrent 15 1 TestProxiedServerServeReadParity 15 1 TestProxiedServerServeRefusesAStreamOnADisabledJournal @@ -309,6 +316,7 @@ 30 10 TestProxiedServerServeEdgeCounts 30 11 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +30 11 TestProxiedServerOutageReconnectAcceptanceMatrix 30 11 TestProxiedServerReady2 30 11 TestProxiedServerServeReadsTheJournalItJustWrote 30 11 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aa5e6472..fce76c527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A proxied-server command against an unreachable external Dolt upstream + now fails within about a second with a clear error instead of stalling + ~20-30s.** The local db proxy stayed up, so the client only saw a bare + close, which the bootstrap ping retried as a transient drop for its whole + 30s budget before reporting `invalid connection`. The proxy now answers such + a connection with a MySQL error (2003, prefixed `beads db proxy:`) saying + what happened: the dial was refused / the socket is missing / the host is + unreachable, or the upstream closed the connection before the MySQL greeting + (down, restarting, or at its connection limit). The client retries that + error only for about a second, so an endpoint that is flapping or rebinding + is still ridden out within one command. A backend that drops a connection + after the greeting, a dial timeout, and every refusal from a managed (local + sidecar) backend keep the full transient retry. - **`bd list` no longer silently drops all but the last repeated filter flag.** `--status`, `--state`, and `--id` were plain string flags, so `bd list --status open --status closed --status pinned` kept only `pinned` — diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6dd0e00a4..437b7411b 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1456,6 +1456,10 @@ sh_test( "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", "BEADS_TEST_PROXIED_SERVER": "1", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + # The external-unix / outage-matrix subtests bridge endpoints with + # socat, which the workers have: fail rather than skip if it is + # missing (requireSocat). The legacy GitHub lane leaves this unset. + "BEADS_TEST_REQUIRE_SOCAT": "1", # The 4-vCPU runner shape: -test.parallel defaults to GOMAXPROCS, and # the job runs 4 tests at once, not one per worker core (39 # concurrent `bd init --proxied-server` against one server hung a diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index 8cfac51f8..8941651af 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -46,6 +46,7 @@ func TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { }}, fixture{name: "external-unix", make: func(t *testing.T) proxiedProject { requireProxiedServerEnv(t) + requireSocat(t) upstream := testutil.StartIsolatedDoltContainerHandle(t) socket := shortSocketPath(t, "dolt.sock") bridge := startHistoryUnixBridge(t, socket, upstream.Port) @@ -316,9 +317,7 @@ func openDirectHistoryDB(t *testing.T, p directHistoryProject) *sql.DB { func startHistoryUnixBridge(t *testing.T, endpoint, upstreamPort string) *exec.Cmd { t.Helper() - if _, err := exec.LookPath("socat"); err != nil { - t.Skipf("socat is required for external Unix topology: %v", err) - } + requireSocat(t) if err := os.Remove(endpoint); err != nil && !os.IsNotExist(err) { t.Fatalf("remove stale Unix socket: %v", err) } diff --git a/cmd/bd/proxied_integration_helpers_test.go b/cmd/bd/proxied_integration_helpers_test.go index cec24780a..ed16e7bcd 100644 --- a/cmd/bd/proxied_integration_helpers_test.go +++ b/cmd/bd/proxied_integration_helpers_test.go @@ -34,6 +34,21 @@ func requireProxiedServerEnv(t *testing.T) { testutil.RequireDoltBinary(t) } +// requireSocat skips a test that needs socat as an external-endpoint bridge +// when it is not installed, or fails it when BEADS_TEST_REQUIRE_SOCAT=1. The +// Bazel proxied lane sets that (its workers have socat), so these subtests +// cannot silently stop running there; the legacy GitHub 15-shard lane has no +// socat and leaves it unset, so they skip there. +func requireSocat(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("socat"); err != nil { + if os.Getenv("BEADS_TEST_REQUIRE_SOCAT") == "1" { + t.Fatalf("socat is required (BEADS_TEST_REQUIRE_SOCAT=1): %v", err) + } + t.Skipf("socat is required: %v", err) + } +} + func bdProxiedEnv(dir string) []string { var env []string for _, e := range os.Environ() { diff --git a/cmd/bd/proxied_outage_reconnect_integration_test.go b/cmd/bd/proxied_outage_reconnect_integration_test.go index 116e55c65..07bed2159 100644 --- a/cmd/bd/proxied_outage_reconnect_integration_test.go +++ b/cmd/bd/proxied_outage_reconnect_integration_test.go @@ -26,14 +26,7 @@ import ( "github.com/stretchr/testify/require" ) -// Deliberately not named TestProxiedServer...: this would make it discovered -// by the shard scripts' ^Test(ProxiedServer|ServerMode) regex, but both its -// subtests still fail a real "exceeded 3s bound" assertion against a stopped -// external upstream (bead filed: outage command exceeds 3s bound against -// stopped external upstream, tcp and unix). Renaming it into a required lane -// before that's fixed would turn Bazel + the legacy 15-shard hash-fallback -// red. Rename it once the underlying latency issue is resolved. -func TestProxiedOutageReconnectAcceptanceMatrix(t *testing.T) { +func TestProxiedServerOutageReconnectAcceptanceMatrix(t *testing.T) { requireProxiedServerEnv(t) bd := buildEmbeddedBD(t) for _, topology := range []struct { @@ -44,10 +37,8 @@ func TestProxiedOutageReconnectAcceptanceMatrix(t *testing.T) { {name: "external-unix-socket", socket: true}, } { t.Run(topology.name, func(t *testing.T) { + requireSocat(t) upstream := testutil.StartIsolatedDoltContainerHandle(t) - if _, err := exec.LookPath("socat"); err != nil { - t.Skip("socat is required") - } gatePort, err := proxy.PickFreePort() require.NoError(t, err) var endpoint string diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 573ab4259..d65b0970e 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -97,7 +97,10 @@ sit behind that API, selected by `BEADS_TEST_DOLT_SERVER`: `BEADS_TEST_REQUIRE_DOLT_CONTAINER=1` turns an unavailable backend into a failure (per test and in every `TestMain`) instead of a skip; lanes that -exist to run the Dolt suites set it. +exist to run the Dolt suites set it. `BEADS_TEST_REQUIRE_SOCAT=1` does the +same for the proxied subtests that bridge an external endpoint with `socat` +(external-unix, the outage/reconnect matrix); `//cmd/bd:bd_proxied_test` +sets it, and the legacy GitHub proxied jobs, which have no `socat`, do not. Under Bazel, `bazel test //... --config=doltserver` runs the Dolt-backed suites of pr.yml's "Test (storage domain + uow)" and "Contract corpus" jobs diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index afdd198ed..5f8cdad9d 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "unverified_process_linux.go", "unverified_process_unsupported.go", "unverified_process_windows.go", + "upstream_error.go", ], importpath = "github.com/steveyegge/beads/internal/storage/dbproxy/proxy", visibility = ["//:__subpackages__"], @@ -36,6 +37,7 @@ go_library( "//internal/storage/dbproxy/server", "//internal/storage/dbproxy/util", "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_go_sql_driver_mysql//:mysql", "@org_golang_x_sync//errgroup", ] + select({ "@rules_go//go/platform:android": [ @@ -70,6 +72,7 @@ go_test( "force_stop_linux_test.go", "server_epoch_internal_test.go", "server_test.go", + "upstream_error_test.go", ], embed = [":proxy"], deps = [ @@ -79,6 +82,7 @@ go_test( "//internal/storage/dbproxy/pidfile", "//internal/storage/dbproxy/server", "//internal/storage/dbproxy/util", + "@com_github_go_sql_driver_mysql//:mysql", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", ] + select({ diff --git a/internal/storage/dbproxy/proxy/external_transport_lifecycle_internal_test.go b/internal/storage/dbproxy/proxy/external_transport_lifecycle_internal_test.go index a07c4f44b..df3ca3fd2 100644 --- a/internal/storage/dbproxy/proxy/external_transport_lifecycle_internal_test.go +++ b/internal/storage/dbproxy/proxy/external_transport_lifecycle_internal_test.go @@ -380,12 +380,16 @@ func TestExternalProxyOutageAndRestoreReDialsTCPAndUnix(t *testing.T) { // Close the restored listener to create a deterministic outage, then // rebind it in the same test so the proxy must re-dial the endpoint. fixture.Close() + // The outage is reported to the client as a MySQL ERR packet in + // place of the greeting (upstream_error.go), then the connection + // closes; no upstream data is ever relayed. failed := dialInternalProxy(t, running.port) _ = failed.SetReadDeadline(time.Now().Add(2 * time.Second)) - var b [1]byte - if _, err := failed.Read(b[:]); err == nil { - t.Fatal("outage client unexpectedly received data") + outage, err := io.ReadAll(failed) + if err != nil { + t.Fatalf("read outage response: %v", err) } + requireUpstreamErrorPacket(t, outage) _ = failed.Close() eventuallyInternal(t, 2*time.Second, func() bool { return running.proxy.stats.Snapshot().BackendDialErrors >= 1 diff --git a/internal/storage/dbproxy/proxy/server.go b/internal/storage/dbproxy/proxy/server.go index 1e0ae8c60..c15d09dcc 100644 --- a/internal/storage/dbproxy/proxy/server.go +++ b/internal/storage/dbproxy/proxy/server.go @@ -56,6 +56,11 @@ type proxyServer struct { stats *Stats stopEpoch string + // reportUpstreamOutage answers a connection whose upstream is not + // serving with a MySQL error instead of a bare close; see + // upstream_error.go. Set for external backends only. + reportUpstreamOutage bool + logger *log.Logger listener net.Listener activeConns atomic.Int64 @@ -101,6 +106,8 @@ func NewProxyServer(opts ProxyOpts) *proxyServer { server: opts.Server, stats: opts.Stats, stopEpoch: opts.StopEpoch, + + reportUpstreamOutage: reportsUpstreamOutage(opts.Server), } } @@ -459,6 +466,9 @@ func (p *proxyServer) handleConn(ctx context.Context, client net.Conn) error { if err != nil { p.tracef("handleConn(%s) backend dial error: %v", addr, err) p.stats.IncBackendDialError() + if p.reportUpstreamOutage && isUpstreamUnreachableDialError(err) { + p.writeUpstreamOutage(client, dialFailureMessage(err)) + } _ = client.Close() return err } @@ -497,6 +507,16 @@ func (p *proxyServer) handleConn(ctx context.Context, client net.Conn) error { n, err := io.Copy(client, backend) p.stats.AddBytesBackendToClient(n) p.tracef("handleConn(%s) backend→client done (n=%d, err=%v)", addr, n, err) + // A MySQL server speaks first, so a backend that reaches EOF having + // sent nothing never served this connection: a front whose own + // target is gone, or a server at its connection limit or shutting + // down. That is not proof of an outage (see upstream_error.go), so + // the client retries the report briefly rather than failing on it. + // A client that hung up first closes backend, which makes this Copy + // fail rather than return a clean EOF, so that case stays silent. + if p.reportUpstreamOutage && n == 0 && err == nil { + p.writeUpstreamOutage(client, closedBeforeGreetingMessage(backend)) + } return err }) return g.Wait() diff --git a/internal/storage/dbproxy/proxy/stats.go b/internal/storage/dbproxy/proxy/stats.go index d7cf2f943..44654fbdc 100644 --- a/internal/storage/dbproxy/proxy/stats.go +++ b/internal/storage/dbproxy/proxy/stats.go @@ -3,19 +3,22 @@ package proxy import "sync" type Counters struct { - ListenAndServeCalls int64 - BackendStartCalls int64 - BackendStopCalls int64 - IdleTimeouts int64 - SignalsReceived int64 - AcceptCalls int64 - AcceptErrors int64 - BackendDialAttempts int64 - BackendDialSuccess int64 - BackendDialErrors int64 - HandledConns int64 - BytesClientToBackend int64 - BytesBackendToClient int64 + ListenAndServeCalls int64 + BackendStartCalls int64 + BackendStopCalls int64 + IdleTimeouts int64 + SignalsReceived int64 + AcceptCalls int64 + AcceptErrors int64 + BackendDialAttempts int64 + BackendDialSuccess int64 + BackendDialErrors int64 + // UpstreamErrorsReported counts connections answered with the + // upstream-outage ERR packet instead of a bare close. + UpstreamErrorsReported int64 + HandledConns int64 + BytesClientToBackend int64 + BytesBackendToClient int64 } type Stats struct { @@ -51,7 +54,10 @@ func (s *Stats) IncAcceptError() { s.update(func(c *Counters) { c.AcceptE func (s *Stats) IncBackendDialAttempt() { s.update(func(c *Counters) { c.BackendDialAttempts++ }) } func (s *Stats) IncBackendDialSuccess() { s.update(func(c *Counters) { c.BackendDialSuccess++ }) } func (s *Stats) IncBackendDialError() { s.update(func(c *Counters) { c.BackendDialErrors++ }) } -func (s *Stats) IncHandledConn() { s.update(func(c *Counters) { c.HandledConns++ }) } +func (s *Stats) IncUpstreamErrorReported() { + s.update(func(c *Counters) { c.UpstreamErrorsReported++ }) +} +func (s *Stats) IncHandledConn() { s.update(func(c *Counters) { c.HandledConns++ }) } func (s *Stats) AddBytesClientToBackend(n int64) { s.update(func(c *Counters) { c.BytesClientToBackend += n }) } diff --git a/internal/storage/dbproxy/proxy/upstream_error.go b/internal/storage/dbproxy/proxy/upstream_error.go new file mode 100644 index 000000000..403e709ea --- /dev/null +++ b/internal/storage/dbproxy/proxy/upstream_error.go @@ -0,0 +1,161 @@ +package proxy + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + "strings" + "syscall" + "time" + "unicode/utf8" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// A client of the proxy only ever sees the proxy's own listener, which is up, +// so a plain close is all it learns when the external upstream behind it is +// not serving: the MySQL driver reports "invalid connection" / unexpected +// EOF, and the uow bootstrap ping classifies that as a transient drop and +// retries it for its whole 30s budget (#6003). For an upstream that is down +// that turns every bd command into a ~20-30s stall ending in an unhelpful +// message. +// +// Instead the proxy answers such a connection itself with a MySQL ERR packet +// in place of the server greeting, naming the upstream and what happened. +// The client (internal/storage/uow, via IsUpstreamOutageError) retries this +// error only for a short window (about a second), long enough to ride out an +// endpoint that is flapping or rebinding, short enough that a real outage +// fails within a couple of seconds instead of thirty. Two shapes qualify: +// +// - The dial was refused, the unix socket path is gone, or the host or +// network is unreachable. Nothing is accepting connections there right +// now. +// - The upstream accepted and then closed without sending a byte. A MySQL +// server speaks first, so it did not serve this connection. That is NOT +// proof it is down: Dolt does exactly this when max_connections and +// back_log are full, when a queued connection times out, and while it +// shuts down for a restart; an L4 front (socat, stunnel, a TCP load +// balancer, kubectl port-forward) does it when its own target is gone. +// The message says so, and the client's short retry window is what +// covers the transient cases. +// +// A dial timeout, a reset, or a connection dropped after the greeting keeps +// the plain close, and with it the client's full transient retry. +// +// External backends only. A managed backend (local Dolt sidecar) is owned by +// the proxy, and its dial can be refused transiently while bd is still +// driving it, which the client's full bootstrap retry absorbs; those keep the +// plain close. +// +// The packet is not byte-for-byte what a real server sends before the +// handshake. MySQL omits the '#'+SQLSTATE marker there (no capabilities are +// negotiated yet); this packet keeps it because dolthub/vitess's client +// always expects it, while go-sql-driver accepts either form. The mysql CLI +// will therefore show "#HY000" at the start of the message text. 2003 +// (CR_CONN_HOST_ERROR, "Can't connect to MySQL server") is a client-side +// code that no real server sends; it is chosen because it describes the +// condition, and the "beads db proxy:" prefix is what marks it as ours. + +const ( + // UpstreamErrorNumber and UpstreamErrorPrefix identify the ERR packet + // the proxy sends for an unreachable external upstream; see + // IsUpstreamOutageError. + UpstreamErrorNumber = 2003 + UpstreamErrorPrefix = "beads db proxy: " + + upstreamErrorSQLState = "HY000" + upstreamErrorMaxMessage = 1024 + upstreamErrorWriteTimeout = time.Second +) + +// IsUpstreamOutageError reports whether err is the proxy's own report that +// its external upstream is unreachable (see the comment above), as opposed to +// an error from the Dolt server itself. +func IsUpstreamOutageError(err error) bool { + var me *mysql.MySQLError + return errors.As(err, &me) && me.Number == UpstreamErrorNumber && + strings.HasPrefix(me.Message, UpstreamErrorPrefix) +} + +// reportsUpstreamOutage reports whether the proxy should answer an +// unreachable upstream with a MySQL error for this backend. +func reportsUpstreamOutage(s server.DatabaseServer) bool { + _, external := s.(*server.ExternalDoltServer) + return external +} + +// isUpstreamUnreachableDialError reports whether a backend dial failure means +// nothing is accepting connections at the upstream right now, as opposed to +// slow (timeout) or a local resource problem. +func isUpstreamUnreachableDialError(err error) bool { + return errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.ENOENT) || + errors.Is(err, syscall.EHOSTUNREACH) || + errors.Is(err, syscall.ENETUNREACH) +} + +// dialFailureMessage describes a refused/unreachable dial as +// "dial
: ", without the Go call-site wrapping the +// error carries. +func dialFailureMessage(err error) string { + var opErr *net.OpError + var errno syscall.Errno + if errors.As(err, &opErr) && opErr.Addr != nil && errors.As(err, &errno) { + return fmt.Sprintf("upstream Dolt server unreachable: %s %s %s: %v", opErr.Op, opErr.Net, opErr.Addr, errno) + } + return "upstream Dolt server unreachable: " + err.Error() +} + +// closedBeforeGreetingMessage describes a backend connection that reached EOF +// before the server greeting, naming the endpoint the proxy dialed. +func closedBeforeGreetingMessage(backend net.Conn) string { + where := "" + if ra := backend.RemoteAddr(); ra != nil && ra.String() != "" { + where = fmt.Sprintf(" at %s %s", ra.Network(), ra) + } + return "upstream Dolt server" + where + " closed the connection before the MySQL greeting " + + "(down, restarting, or at its connection limit)" +} + +// upstreamErrorPacket encodes a MySQL ERR packet (sequence id 0, with the +// '#'+SQLSTATE marker) carrying msg, truncated on a UTF-8 boundary. +func upstreamErrorPacket(msg string) []byte { + if len(msg) > upstreamErrorMaxMessage { + cut := upstreamErrorMaxMessage + for cut > 0 && !utf8.RuneStart(msg[cut]) { + cut-- + } + msg = msg[:cut] + } + payload := make([]byte, 0, 9+len(msg)) + payload = append(payload, 0xff) + payload = binary.LittleEndian.AppendUint16(payload, UpstreamErrorNumber) + payload = append(payload, '#') + payload = append(payload, upstreamErrorSQLState...) + payload = append(payload, msg...) + pkt := make([]byte, 4, 4+len(payload)) + pkt[0] = byte(len(payload)) + pkt[1] = byte(len(payload) >> 8) + pkt[2] = byte(len(payload) >> 16) + pkt[3] = 0 // sequence id: the server's first packet + return append(pkt, payload...) +} + +// UpstreamOutagePacket returns the ERR packet the proxy sends for an +// unreachable external upstream, with detail as the message after +// UpstreamErrorPrefix. Exported so client-side tests can play the proxy. +func UpstreamOutagePacket(detail string) []byte { + return upstreamErrorPacket(UpstreamErrorPrefix + detail) +} + +// writeUpstreamOutage sends the client the proxy's upstream-outage ERR +// packet. Best effort: the caller closes the connection either way. +func (p *proxyServer) writeUpstreamOutage(client net.Conn, detail string) { + p.stats.IncUpstreamErrorReported() + p.tracef("handleConn(%s) reporting upstream outage: %s", client.RemoteAddr(), detail) + _ = client.SetWriteDeadline(time.Now().Add(upstreamErrorWriteTimeout)) + _, _ = client.Write(UpstreamOutagePacket(detail)) +} diff --git a/internal/storage/dbproxy/proxy/upstream_error_test.go b/internal/storage/dbproxy/proxy/upstream_error_test.go new file mode 100644 index 000000000..c50814a9e --- /dev/null +++ b/internal/storage/dbproxy/proxy/upstream_error_test.go @@ -0,0 +1,280 @@ +package proxy + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "log" + "net" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// serveThroughProxy runs handleConn for every connection accepted on a fresh +// loopback listener and returns its address. reportOutage stands in for an +// external backend (the fakes here are not *server.ExternalDoltServer). +func serveThroughProxy(t *testing.T, backend tcpBlackholeBackend, reportOutage bool) string { + t.Helper() + p := NewProxyServer(ProxyOpts{Server: backend, Stats: &Stats{}}) + p.logger = log.Default() + p.reportUpstreamOutage = reportOutage + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { _ = p.handleConn(context.Background(), conn) }() + } + }() + return ln.Addr().String() +} + +// pingThroughProxy pings the proxy with the real MySQL driver and returns the +// error and how long it took. +func pingThroughProxy(t *testing.T, addr string) (time.Duration, error) { + t.Helper() + db, err := sql.Open("mysql", "root@tcp("+addr+")/?timeout=5s") + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + start := time.Now() + err = db.PingContext(ctx) + return time.Since(start), err +} + +func requireUpstreamUnreachable(t *testing.T, err error, elapsed time.Duration, wantInMessage string) { + t.Helper() + var me *mysql.MySQLError + if !errors.As(err, &me) { + t.Fatalf("ping error = %v (%T), want *mysql.MySQLError", err, err) + } + if !IsUpstreamOutageError(err) || string(me.SQLState[:]) != upstreamErrorSQLState { + t.Fatalf("MySQL error = %d/%s %q, want the proxy's %d/%s upstream report", me.Number, me.SQLState, me.Message, UpstreamErrorNumber, upstreamErrorSQLState) + } + if !strings.Contains(me.Message, wantInMessage) { + t.Fatalf("message %q lacks upstream context %q", me.Message, wantInMessage) + } + if elapsed > 2*time.Second { + t.Fatalf("unreachable upstream took %s to report", elapsed) + } +} + +// A refused upstream (nothing listening) is answered with a MySQL error the +// driver reports as permanent, instead of a bare close the uow bootstrap +// would retry for 30s. +func TestHandleConnRefusedUpstreamAnswersWithMySQLError(t *testing.T) { + dead, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + deadAddr := dead.Addr().String() + _ = dead.Close() + + addr := serveThroughProxy(t, tcpBlackholeBackend{address: deadAddr}, true) + elapsed, err := pingThroughProxy(t, addr) + requireUpstreamUnreachable(t, err, elapsed, "connection refused") +} + +// A front that accepts and immediately closes (its own upstream is gone) +// never sends the greeting; the proxy names the endpoint instead of letting +// the client see an unexplained EOF. +func TestHandleConnUpstreamClosingBeforeGreetingAnswersWithMySQLError(t *testing.T) { + front, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = front.Close() }) + go func() { + for { + conn, err := front.Accept() + if err != nil { + return + } + _ = conn.Close() + } + }() + + addr := serveThroughProxy(t, tcpBlackholeBackend{address: front.Addr().String()}, true) + elapsed, err := pingThroughProxy(t, addr) + requireUpstreamUnreachable(t, err, elapsed, front.Addr().String()) + if !strings.Contains(err.Error(), "closed the connection before the MySQL greeting (down, restarting, or at its connection limit)") { + t.Fatalf("zero-byte close must not be reported as a proven outage: %v", err) + } +} + +// A backend that spoke and then dropped the connection is not an unreachable +// upstream: the client must see exactly the backend's bytes and a plain EOF, +// so the bootstrap ping still treats the drop as transient (#6003). +func TestHandleConnUpstreamDropAfterGreetingStaysPlainClose(t *testing.T) { + greeting := []byte("\x01\x00\x00\x00\x0a") + front, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = front.Close() }) + go func() { + conn, err := front.Accept() + if err != nil { + return + } + _, _ = conn.Write(greeting) + _ = conn.Close() + }() + + addr := serveThroughProxy(t, tcpBlackholeBackend{address: front.Addr().String()}, true) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + got, err := io.ReadAll(conn) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != string(greeting) { + t.Fatalf("client received %q, want only the backend's %q", got, greeting) + } +} + +// A managed (non-external) backend keeps the bare close on a refused dial, so +// the client's bootstrap retry still covers a sidecar that is refused +// transiently. +func TestHandleConnManagedBackendRefusalStaysPlainClose(t *testing.T) { + dead, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + deadAddr := dead.Addr().String() + _ = dead.Close() + if reportsUpstreamOutage(tcpBlackholeBackend{address: deadAddr}) { + t.Fatal("a non-external backend must not report upstream outages") + } + + addr := serveThroughProxy(t, tcpBlackholeBackend{address: deadAddr}, false) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + got, err := io.ReadAll(conn) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(got) != 0 { + t.Fatalf("managed backend refusal sent %q, want a bare close", got) + } +} + +// requireUpstreamErrorPacket asserts that got is exactly one MySQL ERR packet +// carrying the proxy's upstream-unreachable error. +func requireUpstreamErrorPacket(t *testing.T, got []byte) { + t.Helper() + if len(got) < 13 { + t.Fatalf("response %q is too short for an ERR packet", got) + } + payloadLen := int(got[0]) | int(got[1])<<8 | int(got[2])<<16 + if payloadLen != len(got)-4 || got[3] != 0 || got[4] != 0xff { + t.Fatalf("response is not a single sequence-0 ERR packet: % x", got) + } + if errno := int(got[5]) | int(got[6])<<8; errno != UpstreamErrorNumber { + t.Fatalf("errno = %d, want %d", errno, UpstreamErrorNumber) + } + if string(got[7:13]) != "#"+upstreamErrorSQLState { + t.Fatalf("SQL state marker = %q, want #%s", got[7:13], upstreamErrorSQLState) + } + if msg := string(got[13:]); !strings.HasPrefix(msg, UpstreamErrorPrefix+"upstream Dolt server") { + t.Fatalf("message %q lacks the proxy prefix and upstream context", msg) + } +} + +// External backends enable the outage report through NewProxyServer; the +// handleConn tests above set the flag by hand on fakes. +func TestNewProxyServerReportsOutagesForExternalBackendsOnly(t *testing.T) { + ext, err := server.NewExternalDoltServer(configfile.ExternalDoltConfig{Host: "127.0.0.1", Port: 3306}) + if err != nil { + t.Fatal(err) + } + if !NewProxyServer(ProxyOpts{Server: ext}).reportUpstreamOutage { + t.Fatal("an external backend must report upstream outages") + } + if NewProxyServer(ProxyOpts{Server: blockingBackend{}}).reportUpstreamOutage { + t.Fatal("a non-external backend must not report upstream outages") + } +} + +// The dial report names network, address and reason, without the Go +// call-site prefixes ExternalDoltServer.Dial wraps the error in. +func TestDialFailureMessageDropsGoCallSites(t *testing.T) { + dead, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + host, portStr, _ := net.SplitHostPort(dead.Addr().String()) + _ = dead.Close() + port, _ := strconv.Atoi(portStr) + ext, err := server.NewExternalDoltServer(configfile.ExternalDoltConfig{Host: host, Port: port}) + if err != nil { + t.Fatal(err) + } + _, dialErr := ext.Dial(context.Background()) + if dialErr == nil || !isUpstreamUnreachableDialError(dialErr) { + t.Fatalf("dial error = %v, want a refusal", dialErr) + } + got := dialFailureMessage(dialErr) + want := "upstream Dolt server unreachable: dial tcp " + dead.Addr().String() + ": connection refused" + if got != want { + t.Fatalf("dialFailureMessage = %q, want %q", got, want) + } +} + +func TestIsUpstreamOutageErrorRequiresTheProxyPrefix(t *testing.T) { + if IsUpstreamOutageError(&mysql.MySQLError{Number: UpstreamErrorNumber, Message: "Can't connect to MySQL server"}) { + t.Fatal("a 2003 without the proxy prefix is not the proxy's report") + } + if IsUpstreamOutageError(&mysql.MySQLError{Number: 1045, Message: UpstreamErrorPrefix + "x"}) { + t.Fatal("only 2003 carries the proxy's report") + } + if !IsUpstreamOutageError(fmt.Errorf("wrapped: %w", &mysql.MySQLError{Number: UpstreamErrorNumber, Message: UpstreamErrorPrefix + "x"})) { + t.Fatal("a wrapped proxy report must be recognized") + } +} + +func TestUpstreamErrorPacketTruncatesOnARuneBoundary(t *testing.T) { + // "é" is two bytes; an odd prefix puts the byte limit mid-rune. + pkt := upstreamErrorPacket("x" + strings.Repeat("é", upstreamErrorMaxMessage)) + if msg := pkt[13:]; !utf8.Valid(msg) || len(msg) > upstreamErrorMaxMessage { + t.Fatalf("truncated message is %d bytes, valid UTF-8 = %v", len(msg), utf8.Valid(msg)) + } +} + +func TestUpstreamErrorPacketTruncatesMessage(t *testing.T) { + pkt := upstreamErrorPacket(strings.Repeat("x", 4*upstreamErrorMaxMessage)) + payloadLen := int(pkt[0]) | int(pkt[1])<<8 | int(pkt[2])<<16 + if payloadLen != len(pkt)-4 || payloadLen != 9+upstreamErrorMaxMessage { + t.Fatalf("payload length %d (packet %d), want %d", payloadLen, len(pkt), 9+upstreamErrorMaxMessage) + } + if pkt[3] != 0 || pkt[4] != 0xff || pkt[7] != '#' { + t.Fatalf("malformed ERR packet header % x", pkt[:8]) + } +} diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 276d3f2e5..99e696215 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -148,6 +148,7 @@ go_test( "testmain_test.go", "tree_walker_contract_test.go", "tx_test.go", + "upstream_outage_window_test.go", "version_reconcile_boot_test.go", "version_reconcile_integration_test.go", "version_reconciler_contract_test.go", diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index e80c119d1..e7681d800 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -607,9 +607,13 @@ const pingAttemptTimeout = 10 * time.Second // application-level rejection that can reach us as one, so at boot an OpError // is always the connection itself: refused while the server restarts, reset, // or broken pipe. MySQL's own rejections arrive as *mysql.MySQLError and stay -// permanent. A name that does not resolve is the exception — it will not start -// resolving, so retrying only spends the whole budget before reporting the -// same misconfiguration. +// permanent here. A name that does not resolve is the exception — it will not +// start resolving, so retrying only spends the whole budget before reporting +// the same misconfiguration. +// +// The db proxy's own upstream-outage report (proxy.IsUpstreamOutageError) is +// a *mysql.MySQLError too, so this function calls it permanent; pingWithRetry +// handles it first and retries it only for upstreamOutageRetryWindow. // // context.DeadlineExceeded counts only because pingWithRetry applies a // per-attempt deadline and checks the caller's context *before* consulting @@ -640,12 +644,45 @@ func isTransientPingError(err error) bool { errors.As(err, &opErr) } +// upstreamOutageRetryWindow is how long pingWithRetry keeps retrying the db +// proxy's report that its external upstream is unreachable (refused, socket +// gone, or closed before the greeting; see dbproxy/proxy/upstream_error.go). +// Long enough to ride out an endpoint that is flapping or rebinding (a Dolt +// restart, a front whose target is being replaced, a server briefly at its +// connection limit), short enough that a real outage fails within a couple of +// seconds instead of spending the full 30s bootstrap budget on a dead +// upstream. A var so tests can shorten it. +var upstreamOutageRetryWindow = time.Second + +// outageWindowBackOff clamps the wrapped backoff so that, once the first +// upstream-outage report has been seen, no wait runs past the end of +// upstreamOutageRetryWindow: the last attempt lands at the window's end +// rather than up to one backoff interval after it. +type outageWindowBackOff struct { + backoff.BackOff + deadline time.Time // zero until the first upstream-outage report +} + +func (b *outageWindowBackOff) NextBackOff() time.Duration { + d := b.BackOff.NextBackOff() + if d == backoff.Stop || b.deadline.IsZero() { + return d + } + if remaining := time.Until(b.deadline); remaining < d { + return max(remaining, 0) + } + return d +} + // pingWithRetry pings until the server answers, the error proves durable, or // bo's MaxElapsedTime is spent. Each attempt gets its own deadline: // backoff.Retry bounds the gap between attempts but cannot interrupt one // already in flight, so without this cap a server that accepts TCP and then -// stalls blocks for the caller's entire context. +// stalls blocks for the caller's entire context. The db proxy's +// upstream-outage report is retried only for upstreamOutageRetryWindow from +// the first time it is seen. func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff, attemptTimeout time.Duration) error { + window := &outageWindowBackOff{BackOff: bo} return backoff.Retry(func() error { attemptCtx, cancel := context.WithTimeout(ctx, attemptTimeout) defer cancel() @@ -661,11 +698,20 @@ func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff if ctx.Err() != nil { return backoff.Permanent(err) } + if proxy.IsUpstreamOutageError(err) { + if window.deadline.IsZero() { + window.deadline = time.Now().Add(upstreamOutageRetryWindow) + } + if !time.Now().Before(window.deadline) { + return backoff.Permanent(err) + } + return err + } if isTransientPingError(err) { return err } return backoff.Permanent(err) - }, backoff.WithContext(bo, ctx)) + }, backoff.WithContext(window, ctx)) } // assertSessionDatabaseOnPool runs assertSessionDatabase on a connection pinned diff --git a/internal/storage/uow/upstream_outage_window_test.go b/internal/storage/uow/upstream_outage_window_test.go new file mode 100644 index 000000000..c1cdcf20e --- /dev/null +++ b/internal/storage/uow/upstream_outage_window_test.go @@ -0,0 +1,215 @@ +package uow + +import ( + "context" + "encoding/binary" + "io" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +func proxyOutageErr() error { + return &mysql.MySQLError{Number: proxy.UpstreamErrorNumber, Message: proxy.UpstreamErrorPrefix + "upstream Dolt server unreachable: dial tcp 127.0.0.1:1: connection refused"} +} + +func shortOutageWindow(t *testing.T, d time.Duration) { + t.Helper() + prev := upstreamOutageRetryWindow + upstreamOutageRetryWindow = d + t.Cleanup(func() { upstreamOutageRetryWindow = prev }) +} + +func TestPingWithRetryRetriesTheProxyOutageReportOnlyBriefly(t *testing.T) { + t.Run("upstream back inside the window recovers", func(t *testing.T) { + shortOutageWindow(t, 2*time.Second) + p := &fakePinger{t: t, errs: []error{proxyOutageErr(), proxyOutageErr(), nil}} + if err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout); err != nil { + t.Fatalf("pingWithRetry() error = %v, want recovery inside the outage window", err) + } + if p.calls != 3 { + t.Fatalf("PingContext called %d times, want 3", p.calls) + } + }) + + t.Run("outage for the whole window fails at its end", func(t *testing.T) { + const window = 300 * time.Millisecond + shortOutageWindow(t, window) + var calls atomic.Int32 + p := pingerFunc(func(context.Context) error { calls.Add(1); return proxyOutageErr() }) + bo := testPingBackOff() + bo.InitialInterval = 50 * time.Millisecond + bo.MaxElapsedTime = 30 * time.Second // the full bootstrap budget must not apply + start := time.Now() + err := pingWithRetry(context.Background(), p, bo, testPingAttemptTimeout) + elapsed := time.Since(start) + if !proxy.IsUpstreamOutageError(err) { + t.Fatalf("pingWithRetry() error = %v, want the proxy's outage report", err) + } + if elapsed < window || elapsed > window+500*time.Millisecond { + t.Fatalf("gave up after %s, want just past the %s window", elapsed, window) + } + if calls.Load() < 2 { + t.Fatalf("PingContext called %d times, want the report retried inside the window", calls.Load()) + } + }) + + t.Run("other transient errors keep the full budget", func(t *testing.T) { + shortOutageWindow(t, time.Nanosecond) + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn, mysql.ErrInvalidConn, nil}} + if err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout); err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil", err) + } + }) +} + +type pingerFunc func(context.Context) error + +func (f pingerFunc) PingContext(ctx context.Context) error { return f(ctx) } + +// TestOpenDBRidesOutAnUpstreamThatReturnsInsideTheOutageWindow plays the db +// proxy in front of an external upstream that is briefly unreachable: the +// first connections get the proxy's real outage packet, and 300ms after the +// first one the upstream "comes back" (a minimal MySQL server answers). One +// openDB call — one bd command's bootstrap — must succeed without the caller +// retrying. +func TestOpenDBRidesOutAnUpstreamThatReturnsInsideTheOutageWindow(t *testing.T) { + front := newFlappingFront(t, 300*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + start := time.Now() + db, err := openDB(ctx, "root@tcp("+front.addr+")/?timeout=5s") + if err != nil { + t.Fatalf("openDB() error = %v, want recovery once the upstream returned", err) + } + _ = db.Close() + if front.refused.Load() == 0 { + t.Fatal("the command never hit the outage; the test proved nothing") + } + if elapsed := time.Since(start); elapsed > upstreamOutageRetryWindow+time.Second { + t.Fatalf("recovery took %s", elapsed) + } +} + +func TestOpenDBFailsFastWhenTheUpstreamStaysUnreachable(t *testing.T) { + front := newFlappingFront(t, time.Hour) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + start := time.Now() + _, err := openDB(ctx, "root@tcp("+front.addr+")/?timeout=5s") + elapsed := time.Since(start) + if !proxy.IsUpstreamOutageError(err) { + t.Fatalf("openDB() error = %v, want the proxy's outage report", err) + } + if elapsed > upstreamOutageRetryWindow+time.Second { + t.Fatalf("an unreachable upstream took %s to fail, want about %s", elapsed, upstreamOutageRetryWindow) + } +} + +// flappingFront answers with the db proxy's outage packet until downFor has +// passed since its first connection, then serves as a minimal MySQL server. +type flappingFront struct { + addr string + refused atomic.Int32 +} + +func newFlappingFront(t *testing.T, downFor time.Duration) *flappingFront { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := &flappingFront{addr: ln.Addr().String()} + var wg sync.WaitGroup + t.Cleanup(func() { _ = ln.Close(); wg.Wait() }) + var once sync.Once + var upAt time.Time + wg.Add(1) + go func() { + defer wg.Done() + for { + conn, err := ln.Accept() + if err != nil { + return + } + once.Do(func() { upAt = time.Now().Add(downFor) }) + if time.Now().Before(upAt) { + f.refused.Add(1) + _, _ = conn.Write(proxy.UpstreamOutagePacket("upstream Dolt server unreachable: dial tcp 127.0.0.1:1: connection refused")) + _ = conn.Close() + continue + } + wg.Add(1) + go func() { defer wg.Done(); serveMinimalMySQL(conn) }() + } + }() + return f +} + +// serveMinimalMySQL speaks just enough of the server side of the MySQL +// protocol for go-sql-driver to connect and ping: a greeting, OK to any +// authentication, and OK to every command until COM_QUIT or EOF. +func serveMinimalMySQL(conn net.Conn) { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + const ( + clientLongPassword = 1 << 0 + clientProtocol41 = 1 << 9 + clientTransactions = 1 << 13 + clientSecureConn = 1 << 15 + clientPluginAuth = 1 << 19 + ) + caps := uint32(clientLongPassword | clientProtocol41 | clientTransactions | clientSecureConn | clientPluginAuth) + g := []byte{0x0a} + g = append(g, "8.0.0-fake\x00"...) + g = binary.LittleEndian.AppendUint32(g, 1) + g = append(g, "abcdefgh"...) // auth data part 1 + g = append(g, 0) + g = binary.LittleEndian.AppendUint16(g, uint16(caps)) + g = append(g, 0x21) // utf8_general_ci + g = binary.LittleEndian.AppendUint16(g, 0x0002) + g = binary.LittleEndian.AppendUint16(g, uint16(caps>>16)) + g = append(g, 21) + g = append(g, make([]byte, 10)...) + g = append(g, "ijklmnopqrst\x00"...) // auth data part 2 + g = append(g, "mysql_native_password\x00"...) + ok := []byte{0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00} + + write := func(seq byte, payload []byte) error { + hdr := []byte{byte(len(payload)), byte(len(payload) >> 8), byte(len(payload) >> 16), seq} + _, err := conn.Write(append(hdr, payload...)) + return err + } + read := func() (byte, []byte, error) { + var hdr [4]byte + if _, err := io.ReadFull(conn, hdr[:]); err != nil { + return 0, nil, err + } + body := make([]byte, int(hdr[0])|int(hdr[1])<<8|int(hdr[2])<<16) + _, err := io.ReadFull(conn, body) + return hdr[3], body, err + } + + if write(0, g) != nil { + return + } + seq, _, err := read() // handshake response + if err != nil || write(seq+1, ok) != nil { + return + } + for { + seq, body, err := read() + if err != nil || len(body) == 0 || body[0] == 0x01 { // EOF or COM_QUIT + return + } + if write(seq+1, ok) != nil { + return + } + } +} diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 168715580..8ebea6c00 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -217,7 +217,9 @@ def render(total, shards, weights): out.append('# Oct 2026 for TestProxiedServerStorageClass,') out.append('# TestProxiedServerServeRefusesAStaleCheckpointWithGone,') out.append('# TestProxiedServerListComments and') - out.append('# TestProxiedServerServeRefusesAStreamOnADisabledJournal).') + out.append('# TestProxiedServerServeRefusesAStreamOnADisabledJournal; and for') + out.append('# TestProxiedServerOutageReconnectAcceptanceMatrix, pinned to shard 1') + out.append('# instead of hashing onto shard 14).') else: out.append(f'# {total}-shard split for the Bazel-only proxied-server tier') out.append('# (bazel-proxied in .github/workflows/bazel.yml), bin-packed') diff --git a/scripts/ci/proxied_test_durations.json b/scripts/ci/proxied_test_durations.json index 2c66ad517..333a2e2b5 100644 --- a/scripts/ci/proxied_test_durations.json +++ b/scripts/ci/proxied_test_durations.json @@ -49,9 +49,16 @@ "//cmd/bd:bd_proxied_test --build_event_json_file=` sweep, then pull", "each shard's bazel-testlogs test.xml per top-level test --", "not from downloaded CI artifacts -- so the parallel=4, CI-representative", - "methodology above is applied consistently." + "methodology above is applied consistently.", + "", + "Added after that sweep: TestProxiedServerOutageReconnectAcceptanceMatrix (80.0,", + "renamed into the discovery pattern once its outage bound was fixed). Not a", + "single sweep's value: local --config=doltserver-proxied runs measured it at", + "55-79 s alone, 60-70 s inside its shard, and a reviewer's 68-109 s under load;", + "80 s is a rounded middle of that spread. seconds_per_init_fallback was", + "recomputed to include it." ], - "seconds_per_init_fallback": 17.1946, + "seconds_per_init_fallback": 17.2952, "tests": { "TestProxiedServerAssign": 38.67, "TestProxiedServerAutoPrune": 67.5, @@ -142,6 +149,7 @@ "TestProxiedServerMultiStatementSQL": 6.76, "TestProxiedServerNote": 11.21, "TestProxiedServerOrphans": 12.11, + "TestProxiedServerOutageReconnectAcceptanceMatrix": 80.0, "TestProxiedServerPathHelpers": 0.0, "TestProxiedServerPing": 11.73, "TestProxiedServerPour": 253.65, diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 54c08748c..94e69ccee 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1739,7 +1739,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { map[string]string{ "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", "BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_GIT_IDENTITY": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", "BEADS_TEST_PROXIED_SERVER": "1", - "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "GOMAXPROCS": "4", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "BEADS_TEST_REQUIRE_SOCAT": "1", "GOMAXPROCS": "4", }, }, "//internal/storage/dolt:dolt_server_conformance_test": {