fix(httpclient): name the connect failure when bd serve cannot be reached (#7702)
* fix(httpclient): name the connect failure when bd serve cannot be reached A workspace whose server does not answer reported the transport's text: `cannot reach bd serve at https://host:30699: Get "https://host:30699/v0/beads/context": dial tcp 100.85.56.42:30699: i/o timeout`. It names the request rather than the address that failed, and says nothing about what a dial timeout usually means. An operator chasing an unreachable store had to know that "i/o timeout" on a dial means the SYN went unanswered. A request whose connection could not be opened now fails with a *wire.DialError inside the *ConnectError: a refused connect names the address and says nothing is listening there; a connect that times out names the address that never answered, how long the attempt waited, and that a firewall dropping the connection or a down host is the usual cause; a host name that does not resolve is named as such. The transport's error stays reachable through Unwrap, so errors.Is checks (ECONNREFUSED, timeouts) and *ConnectError classification are unchanged, and a dial the caller canceled keeps its own error. The events:watch connect gets the same classification. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of gascity-agent Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix(httpclient): keep the transport's dial text in connect errors Review follow-up. The refused and DNS messages dropped "connection refused" and "dial tcp", the vocabulary gc's text classifier matches to tell an unavailable store from other failures; without it the dispatcher's serve loop exits and control beads are quarantined. Every DialError message now ends with the transport's own *net.OpError text in brackets, after the explanation. TestACanceledDialIsTheCallersOwn now actually reaches the ctx.Err() guard: its round tripper cancels and hands back a genuine refused dial, which the guard alone keeps from becoming a DialError (removing the guard fails it). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of gascity-agent Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
4f9174e6ba
commit
18b5c3cc39
7 changed files with 341 additions and 1 deletions
|
|
@ -763,6 +763,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
### Fixed
|
||||
|
||||
- An HTTP workspace whose server cannot be reached now says which step
|
||||
failed. A connect that is refused names the address and says nothing is
|
||||
listening; one that times out (a filtered port, a down host) names the
|
||||
address that never answered and how long it waited; a host name that does
|
||||
not resolve is named as such. The error was the transport's
|
||||
`Get "https://…/v0/beads/context": dial tcp …: i/o timeout`, which named the
|
||||
request rather than the address.
|
||||
- **`bd link` works over a connected HTTP workspace.** It wrote through the
|
||||
store's raw `AddDependencyWithOptions`, which the HTTP client refuses, so
|
||||
every `bd link` failed there with `operation "AddDependencyWithOptions" not
|
||||
|
|
|
|||
|
|
@ -275,7 +275,10 @@ instead of 60s, because `bd serve` gives such a request its
|
|||
`--large-apply-ceiling` (5 minutes by default) and finishes it even after the
|
||||
client has given up. A server whose ceiling is raised past about 5m15s can
|
||||
still outlast the client. A dial or TLS timeout stays a plain connection
|
||||
error, since the server never saw the request. `off` turns off only the
|
||||
error, since the server never saw the request, and a connect that fails says
|
||||
so: it names the address that refused the connection (nothing listening) or
|
||||
never answered (a filtered port or a down host, after the dialer's 30s), or
|
||||
the host name that did not resolve. `off` turns off only the
|
||||
header timer: a reset read is still retried, and a write's timeout is still
|
||||
reported as above.
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ go_library(
|
|||
srcs = [
|
||||
"client.go",
|
||||
"credential.go",
|
||||
"dial_error.go",
|
||||
"doc.go",
|
||||
"handshake.go",
|
||||
"paths.go",
|
||||
|
|
@ -30,6 +31,7 @@ go_test(
|
|||
"client_test.go",
|
||||
"credential_source_test.go",
|
||||
"credential_test.go",
|
||||
"dial_error_test.go",
|
||||
"handshake_test.go",
|
||||
"handshake_wire_revision_test.go",
|
||||
"helpers_test.go",
|
||||
|
|
|
|||
|
|
@ -578,6 +578,7 @@ func (c *Client) roundTripOnce(ctx context.Context, r Request, u *url.URL, stamp
|
|||
attemptCtx, cancel = context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
}
|
||||
started := time.Now()
|
||||
attemptCtx, ht := startHeaderTimer(attemptCtx, headerTimeout)
|
||||
defer ht.done()
|
||||
req := stamped.Clone(attemptCtx)
|
||||
|
|
@ -613,6 +614,9 @@ func (c *Client) roundTripOnce(ctx context.Context, r Request, u *url.URL, stamp
|
|||
hte := &ResponseHeaderTimeoutError{Op: r.Op, Method: r.Method, Host: u.Host, After: time.Since(wroteAt), Attempts: 1, Replayable: r.Replayable(), Err: err}
|
||||
return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: hte}
|
||||
}
|
||||
if de := dialFailure(r.Op, u.Host, err, time.Since(started)); de != nil && ctx.Err() == nil {
|
||||
return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: de}
|
||||
}
|
||||
return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: err}
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
|
|
|||
97
internal/httpclient/wire/dial_error.go
Normal file
97
internal/httpclient/wire/dial_error.go
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
package wire
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DialError is a request that never reached the server because no connection
|
||||
// could be opened: the name did not resolve, or the TCP connect failed. It is
|
||||
// returned inside a *ConnectError, so callers that classify an unreachable
|
||||
// server keep working, and it unwraps to the transport's own error.
|
||||
//
|
||||
// It exists to say which step failed. The transport's text names the request
|
||||
// (`Get "https://…/v0/beads/context": dial tcp …: i/o timeout`) where the
|
||||
// operator needs the address that did not answer and what that usually means.
|
||||
type DialError struct {
|
||||
Op string
|
||||
// Addr is the address the connect was aimed at (ip:port), or the host
|
||||
// name that did not resolve.
|
||||
Addr string
|
||||
// After is how long the attempt ran before it failed.
|
||||
After time.Duration
|
||||
// Resolve reports a name lookup failure rather than a failed connect.
|
||||
Resolve bool
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *DialError) Error() string {
|
||||
switch {
|
||||
case e.Resolve:
|
||||
return fmt.Sprintf("%s: cannot resolve %s: %v (%s)", e.Op, e.Addr, e.cause(), e.transportText())
|
||||
case errors.Is(e.Err, syscall.ECONNREFUSED):
|
||||
return fmt.Sprintf("%s: TCP connection to %s refused: nothing is listening on that port (%s)", e.Op, e.Addr, e.transportText())
|
||||
case isTimeout(e.Err):
|
||||
return fmt.Sprintf("%s: no TCP connection to %s after %s: nothing answered, so a firewall may be dropping the connection or the host is down (%s)",
|
||||
e.Op, e.Addr, e.After.Round(100*time.Millisecond), e.transportText())
|
||||
default:
|
||||
return fmt.Sprintf("%s: TCP connection to %s failed (%s)", e.Op, e.Addr, e.transportText())
|
||||
}
|
||||
}
|
||||
|
||||
// transportText is the transport's own account of the failed dial without the
|
||||
// request it was for: the *net.OpError's text ("dial tcp 10.0.0.1:443:
|
||||
// connect: connection refused"), or the whole error when there is none. Every
|
||||
// message keeps it, because what callers match on is that vocabulary: gc
|
||||
// classifies an unreachable store by "connection refused", "dial tcp" and
|
||||
// "i/o timeout" in bd's error text (dispatch's availability tier, gc hook's
|
||||
// store-unavailable exit), and naming the failed step must not hide it.
|
||||
func (e *DialError) transportText() string {
|
||||
var opErr *net.OpError
|
||||
if errors.As(e.Err, &opErr) {
|
||||
return opErr.Error()
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
// cause is the innermost network error, without the request and address the
|
||||
// transport's own text repeats.
|
||||
func (e *DialError) cause() error {
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(e.Err, &dnsErr) {
|
||||
return errors.New(dnsErr.Err)
|
||||
}
|
||||
var opErr *net.OpError
|
||||
if errors.As(e.Err, &opErr) && opErr.Err != nil {
|
||||
return opErr.Err
|
||||
}
|
||||
return e.Err
|
||||
}
|
||||
|
||||
func (e *DialError) Unwrap() error { return e.Err }
|
||||
|
||||
// Timeout reports whether the connect timed out, so net.Error-style checks
|
||||
// keep classifying it.
|
||||
func (e *DialError) Timeout() bool { return isTimeout(e.Err) }
|
||||
|
||||
// dialFailure classifies a transport error as a resolve or connect failure,
|
||||
// or returns nil for anything else (a TLS handshake, a reset). host is the
|
||||
// request's host:port, named when the transport's error carries no address.
|
||||
func dialFailure(op, host string, err error, after time.Duration) *DialError {
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return &DialError{Op: op, Addr: dnsErr.Name, After: after, Resolve: true, Err: err}
|
||||
}
|
||||
var opErr *net.OpError
|
||||
if !errors.As(err, &opErr) || opErr.Op != "dial" {
|
||||
return nil
|
||||
}
|
||||
addr := host
|
||||
if opErr.Addr != nil {
|
||||
addr = opErr.Addr.String()
|
||||
}
|
||||
return &DialError{Op: op, Addr: addr, After: after, Err: err}
|
||||
}
|
||||
223
internal/httpclient/wire/dial_error_test.go
Normal file
223
internal/httpclient/wire/dial_error_test.go
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
package wire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func dialFailingClient(t *testing.T, base string, dial func(context.Context, string, string) (net.Conn, error)) *Client {
|
||||
t.Helper()
|
||||
u, err := url.Parse(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := Options{ResponseHeaderTimeout: testHeaderTimeout}
|
||||
if dial != nil {
|
||||
opts.HTTPClient = &http.Client{Transport: &http.Transport{DialContext: dial}, Timeout: DefaultTimeout}
|
||||
}
|
||||
c, err := New(u, nil, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func requireDialError(t *testing.T, err error) *DialError {
|
||||
t.Helper()
|
||||
var ce *ConnectError
|
||||
if !errors.As(err, &ce) {
|
||||
t.Fatalf("err = %T %v, want *ConnectError", err, err)
|
||||
}
|
||||
var de *DialError
|
||||
if !errors.As(err, &de) {
|
||||
t.Fatalf("err = %v, want a *DialError inside the *ConnectError", err)
|
||||
}
|
||||
var hte *ResponseHeaderTimeoutError
|
||||
if errors.As(err, &hte) {
|
||||
t.Fatalf("a request that never connected was reported as a header timeout: %v", err)
|
||||
}
|
||||
return de
|
||||
}
|
||||
|
||||
// TestAConnectThatTimesOutNamesTheAddressThatNeverAnswered is the filtered
|
||||
// port: the SYN goes unanswered until the dialer gives up. The error says no
|
||||
// TCP connection was made to which address, and the read is not retried as a
|
||||
// stall, because nothing was ever sent.
|
||||
func TestAConnectThatTimesOutNamesTheAddressThatNeverAnswered(t *testing.T) {
|
||||
dials := 0
|
||||
addr := &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 30699}
|
||||
c := dialFailingClient(t, "https://bd.example:30699", func(context.Context, string, string) (net.Conn, error) {
|
||||
dials++
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Addr: addr, Err: os.ErrDeadlineExceeded}
|
||||
})
|
||||
before := StallRetryCount()
|
||||
err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, nil)
|
||||
de := requireDialError(t, err)
|
||||
if !de.Timeout() {
|
||||
t.Errorf("DialError.Timeout() = false for a connect that timed out")
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, want := range []string{"cannot reach bd serve at https://bd.example:30699", "getContext: no TCP connection to 192.0.2.10:30699", "nothing answered", "i/o timeout"} {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Errorf("error %q does not contain %q", msg, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(msg, "response headers") {
|
||||
t.Errorf("error %q blames response headers for a connect that never completed", msg)
|
||||
}
|
||||
if dials != 1 || StallRetryCount() != before {
|
||||
t.Errorf("dialed %d times with %d stall retries, want one dial and no retry", dials, StallRetryCount()-before)
|
||||
}
|
||||
}
|
||||
|
||||
// TestARefusedConnectSaysNothingIsListening dials a real closed port.
|
||||
func TestARefusedConnectSaysNothingIsListening(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("a Windows refusal is WSAECONNREFUSED, which syscall.ECONNREFUSED does not match; it gets the generic connect wording")
|
||||
}
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := l.Addr().String()
|
||||
_ = l.Close()
|
||||
|
||||
c := dialFailingClient(t, "http://"+addr, nil)
|
||||
err = c.Do(ctx(t), Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
requireDialError(t, err)
|
||||
if !errors.Is(err, syscall.ECONNREFUSED) {
|
||||
t.Errorf("errors.Is(err, ECONNREFUSED) = false; the transport's error must stay reachable: %v", err)
|
||||
}
|
||||
if msg := err.Error(); !strings.Contains(msg, "TCP connection to "+addr+" refused: nothing is listening on that port") {
|
||||
t.Errorf("error %q does not name the refused address", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHostThatDoesNotResolveIsNamed(t *testing.T) {
|
||||
c := dialFailingClient(t, "https://nowhere.invalid", func(context.Context, string, string) (net.Conn, error) {
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{Err: "no such host", Name: "nowhere.invalid", IsNotFound: true}}
|
||||
})
|
||||
err := c.Do(ctx(t), Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
de := requireDialError(t, err)
|
||||
if !de.Resolve {
|
||||
t.Errorf("DialError.Resolve = false for a name that did not resolve")
|
||||
}
|
||||
if msg := err.Error(); !strings.Contains(msg, "cannot resolve nowhere.invalid: no such host") {
|
||||
t.Errorf("error %q does not name the unresolved host", msg)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
// TestACanceledDialIsTheCallersOwn: the caller giving up mid-connect is not a
|
||||
// connect failure to explain, even when what comes back is a genuine dial error
|
||||
// (here a refusal) rather than the cancellation. The round tripper hands the
|
||||
// dial error back directly, as net/http does when the dial loses the race with
|
||||
// the cancel, so the ctx.Err() guard is what keeps it from becoming a DialError.
|
||||
func TestACanceledDialIsTheCallersOwn(t *testing.T) {
|
||||
c, cancel := context.WithCancel(context.Background())
|
||||
addr := &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 30699}
|
||||
u, err := url.Parse("https://bd.example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := New(u, nil, Options{
|
||||
ResponseHeaderTimeout: testHeaderTimeout,
|
||||
HTTPClient: &http.Client{Timeout: DefaultTimeout, Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
cancel()
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Addr: addr, Err: syscall.ECONNREFUSED}
|
||||
})},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = client.Do(c, Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
var de *DialError
|
||||
if errors.As(err, &de) {
|
||||
t.Fatalf("a dial the caller canceled was reported as a connect failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADialSlowerThanTheHeaderTimeoutIsStillAConnectFailure: the header timer
|
||||
// starts once the request is written, so a connect that takes longer than it
|
||||
// and then fails is reported as the connect failure, once, not as missing
|
||||
// headers retried on another connection.
|
||||
func TestADialSlowerThanTheHeaderTimeoutIsStillAConnectFailure(t *testing.T) {
|
||||
dials := 0
|
||||
addr := &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 30699}
|
||||
c := dialFailingClient(t, "https://bd.example:30699", func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
dials++
|
||||
select {
|
||||
case <-time.After(2 * testHeaderTimeout):
|
||||
case <-ctx.Done():
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Addr: addr, Err: ctx.Err()}
|
||||
}
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Addr: addr, Err: os.ErrDeadlineExceeded}
|
||||
})
|
||||
before := StallRetryCount()
|
||||
err := c.Do(ctx(t), Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
requireDialError(t, err)
|
||||
if dials != 1 || StallRetryCount() != before {
|
||||
t.Errorf("dialed %d times with %d stall retries, want one dial and no retry", dials, StallRetryCount()-before)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADialErrorKeepsTheTransportsWords: naming the failed step must not drop
|
||||
// the transport's own text, which callers classify an unreachable server by.
|
||||
// gc's availability tier (internal/dispatch transientNeedles) and gc hook's
|
||||
// store-unavailable markers match "connection refused", "dial tcp" and
|
||||
// "i/o timeout" in bd's error; a refused connect that lost those words would
|
||||
// quarantine a control bead on the first refusal instead of retrying.
|
||||
func TestADialErrorKeepsTheTransportsWords(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("a Windows refusal is worded by WSAECONNREFUSED")
|
||||
}
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := l.Addr().String()
|
||||
_ = l.Close()
|
||||
refused := dialFailingClient(t, "http://"+addr, nil).Do(ctx(t), Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
|
||||
timeoutAddr := &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 30699}
|
||||
timedOut := dialFailingClient(t, "https://bd.example:30699", func(context.Context, string, string) (net.Conn, error) {
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Addr: timeoutAddr, Err: os.ErrDeadlineExceeded}
|
||||
}).Do(ctx(t), Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
|
||||
unresolved := dialFailingClient(t, "https://nowhere.invalid", func(context.Context, string, string) (net.Conn, error) {
|
||||
return nil, &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{Err: "no such host", Name: "nowhere.invalid", IsNotFound: true}}
|
||||
}).Do(ctx(t), Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady}, nil)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
want []string
|
||||
}{
|
||||
{"refused", refused, []string{"refused: nothing is listening on that port", "dial tcp " + addr, "connection refused"}},
|
||||
{"timed out", timedOut, []string{"nothing answered", "dial tcp 192.0.2.10:30699", "i/o timeout"}},
|
||||
{"unresolved", unresolved, []string{"cannot resolve nowhere.invalid", "dial tcp", "no such host"}},
|
||||
} {
|
||||
requireDialError(t, tc.err)
|
||||
msg := tc.err.Error()
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Errorf("%s: error %q does not contain %q", tc.name, msg, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(msg, `Get "`) {
|
||||
t.Errorf("%s: error %q still names the request", tc.name, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -188,10 +188,14 @@ func (c *Client) dialStream(ctx context.Context, hc *http.Client, u *url.URL) (*
|
|||
return nil, err
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
//nolint:gosec // G704: the URL is the workspace's configured server; dialing it IS the feature.
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
cancel()
|
||||
if de := dialFailure(OpWatchEvents, u.Host, err, time.Since(started)); de != nil && ctx.Err() == nil {
|
||||
return nil, &ConnectError{Op: OpWatchEvents, ServerURL: c.base.Redacted(), Err: de}
|
||||
}
|
||||
return nil, &ConnectError{Op: OpWatchEvents, ServerURL: c.base.Redacted(), Err: err}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue