Compare commits

...
Sign in to create a new pull request.

1 commit

Author SHA1 Message Date
Julian Knutsen
639fa01441 fix(dolt): a lost DOLT_COMMIT response in the Deleter is indeterminate, not replayed
The Deleter commits its SQL transaction with the CALL DOLT_COMMIT inside
its withRetryTx body (or withDeleteFence's, for a guarded delete): on a
Dolt sql-server that call commits the transaction, so it is the deletion's
commit point. A failure there was wrapped as a plain error. A lost response
("invalid connection") therefore reached withRetryTx as a pre-commit
connection error, which it replays. The replay re-probed ids the first
attempt had already deleted and reported "issues not found: <id>", a
failure report for a deletion that had landed.

The commit point is now classified with wrapSQLCommitError("dolt commit",
err), as doltAddAndCommitInTx already does for its in-body DOLT_COMMIT. An
undecoded response is ErrCommitIndeterminate and is never replayed. A
decoded server error, such as a 1213 that rolled the transaction back, is
still replayed. The Sweeper has the same in-body commit point under
withWriteTx. It is never replayed, but its lost response was not reported
as indeterminate either; it now is.

Audit: every other withRetryTx/withRetryTxOn body that commits in-body does
so through doltAddAndCommitInTx, which already classifies the response.
The remaining CALL DOLT_COMMIT sites run on a pinned connection after the
SQL transaction, outside any replayed body.

Tests (real Dolt). A MySQL-wire proxy in the test package forwards the
CALL DOLT_COMMIT to the server, then cuts the client off before the
response arrives:
- TestDeleterLostDoltCommitResponseIsIndeterminateAndNotReplayed: the row
  is gone (the delete landed), the error is ErrCommitIndeterminate, and
  the deleter began exactly one transaction. On main it fails with "delete
  error = issues not found: test-doomed". With only the error check
  removed, it also reports "the delete began 2 transactions, want 1".
- TestSweeperLostDoltCommitResponseIsIndeterminate: on main it fails with
  "sweep error = dolt commit: invalid connection".

Both pass with this change. The package's Delete|Deleter|Sweep|Fence|Purge
tests pass too, against a dolt sql-server 2.2.0 test container.

Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-16 00:26:25 +00:00
6 changed files with 342 additions and 2 deletions

View file

@ -189,6 +189,7 @@ go_test(
"crosstier_dep_test.go",
"custom_status_test.go",
"cycle_detector_contract_test.go",
"delete_commit_point_test.go",
"deleter_contract_test.go",
"demote_to_wisp_test.go",
"dependencies_conflict_test.go",
@ -289,6 +290,7 @@ go_test(
"molecule_contract_test.go",
"multistore_multiprocess_test.go",
"open_test.go",
"peer_store_test.go",
"pool_timeouts_test.go",
"port_provenance_test.go",
"portsource_propagation_test.go",
@ -342,6 +344,7 @@ go_test(
"smart_gate_data_behind_test.go",
"smart_remote_migrate_gate_test.go",
"socket_fallback_test.go",
"sql_wire_proxy_test.go",
"stats_reporter_contract_test.go",
"store_branch_pin_test.go",
"store_unit_test.go",

View file

@ -0,0 +1,95 @@
package dolt
import (
"errors"
"strings"
"testing"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
"github.com/steveyegge/beads/issueops"
)
// The Deleter and the Sweeper commit their SQL transaction with the CALL
// DOLT_COMMIT inside their transaction body: that statement is their commit
// point. A response lost there may mean the deletion landed, so it must surface
// as ErrCommitIndeterminate, and withRetryTx must not replay a deletion that
// may already be durable.
func isDoltCommitCall(query string) bool {
return strings.HasPrefix(strings.ToUpper(query), "CALL DOLT_COMMIT")
}
func isTransactionBegin(query string) bool {
return strings.HasPrefix(strings.ToUpper(query), "START TRANSACTION")
}
func TestDeleterLostDoltCommitResponseIsIndeterminateAndNotReplayed(t *testing.T) {
base, cleanup := setupConcurrentTestStore(t)
defer cleanup()
ctx, cancel := concurrentTestContext(t)
defer cancel()
if err := base.CreateIssue(ctx, &types.Issue{ID: "test-doomed", Title: "doomed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil {
t.Fatalf("seed: %v", err)
}
proxy := startSQLWireProxy(t)
via := openStoreOnPort(t, ctx, base, proxy.port())
deleter, err := via.Deleter()
if err != nil {
t.Fatalf("Deleter(): %v", err)
}
mark := proxy.mark()
sprung := proxy.dropResponseToNext(isDoltCommitCall)
_, err = deleter.Delete(ctx, issueops.DeleteRequest{Actor: "deleter", IDs: []string{"test-doomed"}, Force: true})
select {
case <-sprung:
default:
t.Fatalf("the delete never reached CALL DOLT_COMMIT (err = %v)", err)
}
if _, gerr := base.GetIssue(ctx, "test-doomed"); !errors.Is(gerr, storage.ErrNotFound) {
t.Fatalf("read test-doomed after the lost response = %v, want ErrNotFound: the delete did not land, so the test exercised no lost response", gerr)
}
if err == nil {
t.Fatal("a lost DOLT_COMMIT response reported success")
}
if !errors.Is(err, ErrCommitIndeterminate) {
t.Fatalf("delete error = %v, want ErrCommitIndeterminate: the deletion may have landed", err)
}
if n := proxy.statementsSince(mark, isTransactionBegin); n != 1 {
t.Fatalf("the delete began %d transactions, want 1: a deletion that may have landed was replayed", n)
}
}
func TestSweeperLostDoltCommitResponseIsIndeterminate(t *testing.T) {
base, cleanup := setupConcurrentTestStore(t)
defer cleanup()
ctx, cancel := concurrentTestContext(t)
defer cancel()
if err := base.CreateIssue(ctx, &types.Issue{ID: "test-swept", Title: "swept", Status: types.StatusClosed, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil {
t.Fatalf("seed: %v", err)
}
proxy := startSQLWireProxy(t)
via := openStoreOnPort(t, ctx, base, proxy.port())
sweeper, err := via.Sweeper()
if err != nil {
t.Fatalf("Sweeper(): %v", err)
}
sprung := proxy.dropResponseToNext(isDoltCommitCall)
_, err = sweeper.Sweep(ctx, issueops.SweepRequest{Actor: "sweeper", Tier: issueops.SweepDurable, IDPattern: "test-swept"})
select {
case <-sprung:
default:
t.Fatalf("the sweep never reached CALL DOLT_COMMIT (err = %v)", err)
}
if _, gerr := base.GetIssue(ctx, "test-swept"); !errors.Is(gerr, storage.ErrNotFound) {
t.Fatalf("read test-swept after the lost response = %v, want ErrNotFound: the sweep did not land, so the test exercised no lost response", gerr)
}
if err == nil {
t.Fatal("a lost DOLT_COMMIT response reported success")
}
if !errors.Is(err, ErrCommitIndeterminate) {
t.Fatalf("sweep error = %v, want ErrCommitIndeterminate: the sweep may have landed", err)
}
}

View file

@ -103,7 +103,10 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue
msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted)
if err := schema.DrainCall(ctx, tx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)",
msg, s.store.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) {
return fmt.Errorf("dolt commit: %w", err)
// DOLT_COMMIT commits this SQL transaction: it is the deletion's
// commit point, so a lost response is indeterminate and must not
// reach withRetryTx as a pre-commit connection error it replays.
return wrapSQLCommitError("dolt commit", err)
}
return nil
}

View file

@ -0,0 +1,33 @@
package dolt
import (
"context"
"testing"
)
// openPeerStore opens a second store on store's database, with a connection
// pool of its own as a second bd process would have.
func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore {
t.Helper()
return openStoreOnPort(t, ctx, store, testServerPort)
}
// openStoreOnPort opens a store on store's database through whatever listens
// on port: the test server itself, or a test proxy in front of it.
func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore {
t.Helper()
peer, err := New(ctx, &Config{
Path: t.TempDir(),
CommitterName: "test",
CommitterEmail: "test@example.com",
ServerHost: "127.0.0.1",
ServerPort: port,
Database: store.database,
MaxOpenConns: 2,
})
if err != nil {
t.Fatalf("open store on %s through port %d: %v", store.database, port, err)
}
t.Cleanup(func() { _ = peer.Close() })
return peer
}

View file

@ -0,0 +1,205 @@
package dolt
import (
"fmt"
"io"
"net"
"strings"
"sync"
"sync/atomic"
"testing"
)
// comQuery is the MySQL command byte of a text-protocol statement. Every
// statement a store sends travels as one: its DSN interpolates parameters.
const comQuery = 0x03
// sqlWireProxy forwards MySQL wire traffic between stores under test and the
// test Dolt server. It records every statement it forwards and can trap the
// next one that matches a predicate, either holding it until the test releases
// it (to order two writers' commits deterministically) or forwarding it and
// cutting the client off before the server's answer arrives (a response lost
// after the server acted on the statement). Production code needs no test hook.
type sqlWireProxy struct {
ln net.Listener
target string
mu sync.Mutex
trap *sqlWireTrap
held []*sqlWireTrap
queries []string
conns []net.Conn
}
type sqlWireTrap struct {
match func(query string) bool
drop bool
sprung chan struct{}
release chan struct{}
released sync.Once
}
func (trap *sqlWireTrap) open() { trap.released.Do(func() { close(trap.release) }) }
func startSQLWireProxy(t *testing.T) *sqlWireProxy {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("proxy listen: %v", err)
}
p := &sqlWireProxy{ln: ln, target: fmt.Sprintf("127.0.0.1:%d", testServerPort)}
go func() {
for {
client, err := ln.Accept()
if err != nil {
return
}
go p.handle(client)
}
}()
t.Cleanup(p.close)
return p
}
func (p *sqlWireProxy) port() int { return p.ln.Addr().(*net.TCPAddr).Port }
// holdNext traps the next statement matching match and keeps it from the
// server until release is called. sprung is closed once it is caught.
func (p *sqlWireProxy) holdNext(match func(query string) bool) (sprung <-chan struct{}, release func()) {
trap := p.arm(match, false)
return trap.sprung, trap.open
}
// dropResponseToNext traps the next statement matching match: the server runs
// it, and the client's connection is cut instead of receiving the answer.
func (p *sqlWireProxy) dropResponseToNext(match func(query string) bool) (sprung <-chan struct{}) {
return p.arm(match, true).sprung
}
func (p *sqlWireProxy) arm(match func(query string) bool, drop bool) *sqlWireTrap {
trap := &sqlWireTrap{match: match, drop: drop, sprung: make(chan struct{}), release: make(chan struct{})}
p.mu.Lock()
defer p.mu.Unlock()
if p.trap != nil {
p.trap.open()
}
p.trap = trap
return trap
}
// mark returns a position in the statement log for statementsSince.
func (p *sqlWireProxy) mark() int {
p.mu.Lock()
defer p.mu.Unlock()
return len(p.queries)
}
// statementsSince counts the statements forwarded after mark that match.
func (p *sqlWireProxy) statementsSince(mark int, match func(query string) bool) int {
p.mu.Lock()
defer p.mu.Unlock()
n := 0
for _, q := range p.queries[mark:] {
if match(q) {
n++
}
}
return n
}
// intercept logs query and hands back the armed trap when it catches query.
func (p *sqlWireProxy) intercept(query string) *sqlWireTrap {
p.mu.Lock()
defer p.mu.Unlock()
p.queries = append(p.queries, query)
trap := p.trap
if trap == nil || !trap.match(query) {
return nil
}
p.trap = nil
if !trap.drop {
p.held = append(p.held, trap)
}
close(trap.sprung)
return trap
}
func (p *sqlWireProxy) close() {
_ = p.ln.Close()
p.mu.Lock()
defer p.mu.Unlock()
if p.trap != nil {
p.trap.open()
}
for _, trap := range p.held {
trap.open()
}
for _, c := range p.conns {
_ = c.Close()
}
}
func (p *sqlWireProxy) handle(client net.Conn) {
server, err := net.Dial("tcp", p.target)
if err != nil {
_ = client.Close()
return
}
p.mu.Lock()
p.conns = append(p.conns, client, server)
p.mu.Unlock()
var once sync.Once
closeBoth := func() { once.Do(func() { _ = client.Close(); _ = server.Close() }) }
var dropping atomic.Bool
go func() {
buf := make([]byte, 64*1024)
for {
n, err := server.Read(buf)
if n > 0 {
if dropping.Load() {
closeBoth()
return
}
if _, werr := client.Write(buf[:n]); werr != nil {
closeBoth()
return
}
}
if err != nil {
closeBoth()
return
}
}
}()
hdr := make([]byte, 4)
for {
if _, err := io.ReadFull(client, hdr); err != nil {
closeBoth()
return
}
n := int(hdr[0]) | int(hdr[1])<<8 | int(hdr[2])<<16
packet := make([]byte, 4+n)
copy(packet, hdr)
if _, err := io.ReadFull(client, packet[4:]); err != nil {
closeBoth()
return
}
// A command starts a new exchange (sequence id 0); the handshake's
// packets do not.
if hdr[3] == 0 && n > 1 && packet[4] == comQuery {
if trap := p.intercept(strings.TrimSpace(string(packet[5:]))); trap != nil {
if trap.drop {
dropping.Store(true)
} else {
<-trap.release
}
}
}
if _, err := server.Write(packet); err != nil {
closeBoth()
return
}
}
}
func isCommitStatement(query string) bool { return strings.EqualFold(query, "COMMIT") }

View file

@ -86,7 +86,8 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop
msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier)
if err := schema.DrainCall(ctx, tx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)",
msg, s.store.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) {
return fmt.Errorf("dolt commit: %w", err)
// The sweep's commit point, as for the Deleter.
return wrapSQLCommitError("dolt commit", err)
}
return nil
}); err != nil {