diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2be809cc9..0ea4187f8 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -189,6 +189,7 @@ go_test( "crosstier_dep_test.go", "custom_status_test.go", "cycle_detector_contract_test.go", + "delete_commit_point_test.go", "deleter_contract_test.go", "demote_to_wisp_test.go", "dependencies_conflict_test.go", @@ -289,6 +290,7 @@ go_test( "molecule_contract_test.go", "multistore_multiprocess_test.go", "open_test.go", + "peer_store_test.go", "pool_timeouts_test.go", "port_provenance_test.go", "portsource_propagation_test.go", @@ -342,6 +344,7 @@ go_test( "smart_gate_data_behind_test.go", "smart_remote_migrate_gate_test.go", "socket_fallback_test.go", + "sql_wire_proxy_test.go", "stats_reporter_contract_test.go", "store_branch_pin_test.go", "store_unit_test.go", diff --git a/internal/storage/dolt/delete_commit_point_test.go b/internal/storage/dolt/delete_commit_point_test.go new file mode 100644 index 000000000..38eb0b766 --- /dev/null +++ b/internal/storage/dolt/delete_commit_point_test.go @@ -0,0 +1,95 @@ +package dolt + +import ( + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Deleter and the Sweeper commit their SQL transaction with the CALL +// DOLT_COMMIT inside their transaction body: that statement is their commit +// point. A response lost there may mean the deletion landed, so it must surface +// as ErrCommitIndeterminate, and withRetryTx must not replay a deletion that +// may already be durable. + +func isDoltCommitCall(query string) bool { + return strings.HasPrefix(strings.ToUpper(query), "CALL DOLT_COMMIT") +} + +func isTransactionBegin(query string) bool { + return strings.HasPrefix(strings.ToUpper(query), "START TRANSACTION") +} + +func TestDeleterLostDoltCommitResponseIsIndeterminateAndNotReplayed(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + if err := base.CreateIssue(ctx, &types.Issue{ID: "test-doomed", Title: "doomed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + proxy := startSQLWireProxy(t) + via := openStoreOnPort(t, ctx, base, proxy.port()) + deleter, err := via.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + + mark := proxy.mark() + sprung := proxy.dropResponseToNext(isDoltCommitCall) + _, err = deleter.Delete(ctx, issueops.DeleteRequest{Actor: "deleter", IDs: []string{"test-doomed"}, Force: true}) + select { + case <-sprung: + default: + t.Fatalf("the delete never reached CALL DOLT_COMMIT (err = %v)", err) + } + if _, gerr := base.GetIssue(ctx, "test-doomed"); !errors.Is(gerr, storage.ErrNotFound) { + t.Fatalf("read test-doomed after the lost response = %v, want ErrNotFound: the delete did not land, so the test exercised no lost response", gerr) + } + if err == nil { + t.Fatal("a lost DOLT_COMMIT response reported success") + } + if !errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("delete error = %v, want ErrCommitIndeterminate: the deletion may have landed", err) + } + if n := proxy.statementsSince(mark, isTransactionBegin); n != 1 { + t.Fatalf("the delete began %d transactions, want 1: a deletion that may have landed was replayed", n) + } +} + +func TestSweeperLostDoltCommitResponseIsIndeterminate(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + if err := base.CreateIssue(ctx, &types.Issue{ID: "test-swept", Title: "swept", Status: types.StatusClosed, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + proxy := startSQLWireProxy(t) + via := openStoreOnPort(t, ctx, base, proxy.port()) + sweeper, err := via.Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + + sprung := proxy.dropResponseToNext(isDoltCommitCall) + _, err = sweeper.Sweep(ctx, issueops.SweepRequest{Actor: "sweeper", Tier: issueops.SweepDurable, IDPattern: "test-swept"}) + select { + case <-sprung: + default: + t.Fatalf("the sweep never reached CALL DOLT_COMMIT (err = %v)", err) + } + if _, gerr := base.GetIssue(ctx, "test-swept"); !errors.Is(gerr, storage.ErrNotFound) { + t.Fatalf("read test-swept after the lost response = %v, want ErrNotFound: the sweep did not land, so the test exercised no lost response", gerr) + } + if err == nil { + t.Fatal("a lost DOLT_COMMIT response reported success") + } + if !errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("sweep error = %v, want ErrCommitIndeterminate: the sweep may have landed", err) + } +} diff --git a/internal/storage/dolt/deleter.go b/internal/storage/dolt/deleter.go index 59cbd811b..011c5c533 100644 --- a/internal/storage/dolt/deleter.go +++ b/internal/storage/dolt/deleter.go @@ -103,7 +103,10 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted) if err := schema.DrainCall(ctx, tx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", msg, s.store.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { - return fmt.Errorf("dolt commit: %w", err) + // DOLT_COMMIT commits this SQL transaction: it is the deletion's + // commit point, so a lost response is indeterminate and must not + // reach withRetryTx as a pre-commit connection error it replays. + return wrapSQLCommitError("dolt commit", err) } return nil } diff --git a/internal/storage/dolt/peer_store_test.go b/internal/storage/dolt/peer_store_test.go new file mode 100644 index 000000000..9a6d6653e --- /dev/null +++ b/internal/storage/dolt/peer_store_test.go @@ -0,0 +1,33 @@ +package dolt + +import ( + "context" + "testing" +) + +// openPeerStore opens a second store on store's database, with a connection +// pool of its own as a second bd process would have. +func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore { + t.Helper() + return openStoreOnPort(t, ctx, store, testServerPort) +} + +// openStoreOnPort opens a store on store's database through whatever listens +// on port: the test server itself, or a test proxy in front of it. +func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore { + t.Helper() + peer, err := New(ctx, &Config{ + Path: t.TempDir(), + CommitterName: "test", + CommitterEmail: "test@example.com", + ServerHost: "127.0.0.1", + ServerPort: port, + Database: store.database, + MaxOpenConns: 2, + }) + if err != nil { + t.Fatalf("open store on %s through port %d: %v", store.database, port, err) + } + t.Cleanup(func() { _ = peer.Close() }) + return peer +} diff --git a/internal/storage/dolt/sql_wire_proxy_test.go b/internal/storage/dolt/sql_wire_proxy_test.go new file mode 100644 index 000000000..98e40b518 --- /dev/null +++ b/internal/storage/dolt/sql_wire_proxy_test.go @@ -0,0 +1,205 @@ +package dolt + +import ( + "fmt" + "io" + "net" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// comQuery is the MySQL command byte of a text-protocol statement. Every +// statement a store sends travels as one: its DSN interpolates parameters. +const comQuery = 0x03 + +// sqlWireProxy forwards MySQL wire traffic between stores under test and the +// test Dolt server. It records every statement it forwards and can trap the +// next one that matches a predicate, either holding it until the test releases +// it (to order two writers' commits deterministically) or forwarding it and +// cutting the client off before the server's answer arrives (a response lost +// after the server acted on the statement). Production code needs no test hook. +type sqlWireProxy struct { + ln net.Listener + target string + + mu sync.Mutex + trap *sqlWireTrap + held []*sqlWireTrap + queries []string + conns []net.Conn +} + +type sqlWireTrap struct { + match func(query string) bool + drop bool + sprung chan struct{} + release chan struct{} + released sync.Once +} + +func (trap *sqlWireTrap) open() { trap.released.Do(func() { close(trap.release) }) } + +func startSQLWireProxy(t *testing.T) *sqlWireProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + p := &sqlWireProxy{ln: ln, target: fmt.Sprintf("127.0.0.1:%d", testServerPort)} + go func() { + for { + client, err := ln.Accept() + if err != nil { + return + } + go p.handle(client) + } + }() + t.Cleanup(p.close) + return p +} + +func (p *sqlWireProxy) port() int { return p.ln.Addr().(*net.TCPAddr).Port } + +// holdNext traps the next statement matching match and keeps it from the +// server until release is called. sprung is closed once it is caught. +func (p *sqlWireProxy) holdNext(match func(query string) bool) (sprung <-chan struct{}, release func()) { + trap := p.arm(match, false) + return trap.sprung, trap.open +} + +// dropResponseToNext traps the next statement matching match: the server runs +// it, and the client's connection is cut instead of receiving the answer. +func (p *sqlWireProxy) dropResponseToNext(match func(query string) bool) (sprung <-chan struct{}) { + return p.arm(match, true).sprung +} + +func (p *sqlWireProxy) arm(match func(query string) bool, drop bool) *sqlWireTrap { + trap := &sqlWireTrap{match: match, drop: drop, sprung: make(chan struct{}), release: make(chan struct{})} + p.mu.Lock() + defer p.mu.Unlock() + if p.trap != nil { + p.trap.open() + } + p.trap = trap + return trap +} + +// mark returns a position in the statement log for statementsSince. +func (p *sqlWireProxy) mark() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.queries) +} + +// statementsSince counts the statements forwarded after mark that match. +func (p *sqlWireProxy) statementsSince(mark int, match func(query string) bool) int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, q := range p.queries[mark:] { + if match(q) { + n++ + } + } + return n +} + +// intercept logs query and hands back the armed trap when it catches query. +func (p *sqlWireProxy) intercept(query string) *sqlWireTrap { + p.mu.Lock() + defer p.mu.Unlock() + p.queries = append(p.queries, query) + trap := p.trap + if trap == nil || !trap.match(query) { + return nil + } + p.trap = nil + if !trap.drop { + p.held = append(p.held, trap) + } + close(trap.sprung) + return trap +} + +func (p *sqlWireProxy) close() { + _ = p.ln.Close() + p.mu.Lock() + defer p.mu.Unlock() + if p.trap != nil { + p.trap.open() + } + for _, trap := range p.held { + trap.open() + } + for _, c := range p.conns { + _ = c.Close() + } +} + +func (p *sqlWireProxy) handle(client net.Conn) { + server, err := net.Dial("tcp", p.target) + if err != nil { + _ = client.Close() + return + } + p.mu.Lock() + p.conns = append(p.conns, client, server) + p.mu.Unlock() + var once sync.Once + closeBoth := func() { once.Do(func() { _ = client.Close(); _ = server.Close() }) } + var dropping atomic.Bool + go func() { + buf := make([]byte, 64*1024) + for { + n, err := server.Read(buf) + if n > 0 { + if dropping.Load() { + closeBoth() + return + } + if _, werr := client.Write(buf[:n]); werr != nil { + closeBoth() + return + } + } + if err != nil { + closeBoth() + return + } + } + }() + hdr := make([]byte, 4) + for { + if _, err := io.ReadFull(client, hdr); err != nil { + closeBoth() + return + } + n := int(hdr[0]) | int(hdr[1])<<8 | int(hdr[2])<<16 + packet := make([]byte, 4+n) + copy(packet, hdr) + if _, err := io.ReadFull(client, packet[4:]); err != nil { + closeBoth() + return + } + // A command starts a new exchange (sequence id 0); the handshake's + // packets do not. + if hdr[3] == 0 && n > 1 && packet[4] == comQuery { + if trap := p.intercept(strings.TrimSpace(string(packet[5:]))); trap != nil { + if trap.drop { + dropping.Store(true) + } else { + <-trap.release + } + } + } + if _, err := server.Write(packet); err != nil { + closeBoth() + return + } + } +} + +func isCommitStatement(query string) bool { return strings.EqualFold(query, "COMMIT") } diff --git a/internal/storage/dolt/sweeper.go b/internal/storage/dolt/sweeper.go index d1f6d81d8..71a6137f8 100644 --- a/internal/storage/dolt/sweeper.go +++ b/internal/storage/dolt/sweeper.go @@ -86,7 +86,8 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier) if err := schema.DrainCall(ctx, tx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", msg, s.store.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { - return fmt.Errorf("dolt commit: %w", err) + // The sweep's commit point, as for the Deleter. + return wrapSQLCommitError("dolt commit", err) } return nil }); err != nil {