Compare commits
1 commit
main
...
fix/delete
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
639fa01441 |
6 changed files with 342 additions and 2 deletions
|
|
@ -189,6 +189,7 @@ go_test(
|
|||
"crosstier_dep_test.go",
|
||||
"custom_status_test.go",
|
||||
"cycle_detector_contract_test.go",
|
||||
"delete_commit_point_test.go",
|
||||
"deleter_contract_test.go",
|
||||
"demote_to_wisp_test.go",
|
||||
"dependencies_conflict_test.go",
|
||||
|
|
@ -289,6 +290,7 @@ go_test(
|
|||
"molecule_contract_test.go",
|
||||
"multistore_multiprocess_test.go",
|
||||
"open_test.go",
|
||||
"peer_store_test.go",
|
||||
"pool_timeouts_test.go",
|
||||
"port_provenance_test.go",
|
||||
"portsource_propagation_test.go",
|
||||
|
|
@ -342,6 +344,7 @@ go_test(
|
|||
"smart_gate_data_behind_test.go",
|
||||
"smart_remote_migrate_gate_test.go",
|
||||
"socket_fallback_test.go",
|
||||
"sql_wire_proxy_test.go",
|
||||
"stats_reporter_contract_test.go",
|
||||
"store_branch_pin_test.go",
|
||||
"store_unit_test.go",
|
||||
|
|
|
|||
95
internal/storage/dolt/delete_commit_point_test.go
Normal file
95
internal/storage/dolt/delete_commit_point_test.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
"github.com/steveyegge/beads/issueops"
|
||||
)
|
||||
|
||||
// The Deleter and the Sweeper commit their SQL transaction with the CALL
|
||||
// DOLT_COMMIT inside their transaction body: that statement is their commit
|
||||
// point. A response lost there may mean the deletion landed, so it must surface
|
||||
// as ErrCommitIndeterminate, and withRetryTx must not replay a deletion that
|
||||
// may already be durable.
|
||||
|
||||
func isDoltCommitCall(query string) bool {
|
||||
return strings.HasPrefix(strings.ToUpper(query), "CALL DOLT_COMMIT")
|
||||
}
|
||||
|
||||
func isTransactionBegin(query string) bool {
|
||||
return strings.HasPrefix(strings.ToUpper(query), "START TRANSACTION")
|
||||
}
|
||||
|
||||
func TestDeleterLostDoltCommitResponseIsIndeterminateAndNotReplayed(t *testing.T) {
|
||||
base, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
if err := base.CreateIssue(ctx, &types.Issue{ID: "test-doomed", Title: "doomed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
proxy := startSQLWireProxy(t)
|
||||
via := openStoreOnPort(t, ctx, base, proxy.port())
|
||||
deleter, err := via.Deleter()
|
||||
if err != nil {
|
||||
t.Fatalf("Deleter(): %v", err)
|
||||
}
|
||||
|
||||
mark := proxy.mark()
|
||||
sprung := proxy.dropResponseToNext(isDoltCommitCall)
|
||||
_, err = deleter.Delete(ctx, issueops.DeleteRequest{Actor: "deleter", IDs: []string{"test-doomed"}, Force: true})
|
||||
select {
|
||||
case <-sprung:
|
||||
default:
|
||||
t.Fatalf("the delete never reached CALL DOLT_COMMIT (err = %v)", err)
|
||||
}
|
||||
if _, gerr := base.GetIssue(ctx, "test-doomed"); !errors.Is(gerr, storage.ErrNotFound) {
|
||||
t.Fatalf("read test-doomed after the lost response = %v, want ErrNotFound: the delete did not land, so the test exercised no lost response", gerr)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("a lost DOLT_COMMIT response reported success")
|
||||
}
|
||||
if !errors.Is(err, ErrCommitIndeterminate) {
|
||||
t.Fatalf("delete error = %v, want ErrCommitIndeterminate: the deletion may have landed", err)
|
||||
}
|
||||
if n := proxy.statementsSince(mark, isTransactionBegin); n != 1 {
|
||||
t.Fatalf("the delete began %d transactions, want 1: a deletion that may have landed was replayed", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweeperLostDoltCommitResponseIsIndeterminate(t *testing.T) {
|
||||
base, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
if err := base.CreateIssue(ctx, &types.Issue{ID: "test-swept", Title: "swept", Status: types.StatusClosed, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
proxy := startSQLWireProxy(t)
|
||||
via := openStoreOnPort(t, ctx, base, proxy.port())
|
||||
sweeper, err := via.Sweeper()
|
||||
if err != nil {
|
||||
t.Fatalf("Sweeper(): %v", err)
|
||||
}
|
||||
|
||||
sprung := proxy.dropResponseToNext(isDoltCommitCall)
|
||||
_, err = sweeper.Sweep(ctx, issueops.SweepRequest{Actor: "sweeper", Tier: issueops.SweepDurable, IDPattern: "test-swept"})
|
||||
select {
|
||||
case <-sprung:
|
||||
default:
|
||||
t.Fatalf("the sweep never reached CALL DOLT_COMMIT (err = %v)", err)
|
||||
}
|
||||
if _, gerr := base.GetIssue(ctx, "test-swept"); !errors.Is(gerr, storage.ErrNotFound) {
|
||||
t.Fatalf("read test-swept after the lost response = %v, want ErrNotFound: the sweep did not land, so the test exercised no lost response", gerr)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("a lost DOLT_COMMIT response reported success")
|
||||
}
|
||||
if !errors.Is(err, ErrCommitIndeterminate) {
|
||||
t.Fatalf("sweep error = %v, want ErrCommitIndeterminate: the sweep may have landed", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -103,7 +103,10 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue
|
|||
msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted)
|
||||
if err := schema.DrainCall(ctx, tx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)",
|
||||
msg, s.store.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) {
|
||||
return fmt.Errorf("dolt commit: %w", err)
|
||||
// DOLT_COMMIT commits this SQL transaction: it is the deletion's
|
||||
// commit point, so a lost response is indeterminate and must not
|
||||
// reach withRetryTx as a pre-commit connection error it replays.
|
||||
return wrapSQLCommitError("dolt commit", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
33
internal/storage/dolt/peer_store_test.go
Normal file
33
internal/storage/dolt/peer_store_test.go
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// openPeerStore opens a second store on store's database, with a connection
|
||||
// pool of its own as a second bd process would have.
|
||||
func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore {
|
||||
t.Helper()
|
||||
return openStoreOnPort(t, ctx, store, testServerPort)
|
||||
}
|
||||
|
||||
// openStoreOnPort opens a store on store's database through whatever listens
|
||||
// on port: the test server itself, or a test proxy in front of it.
|
||||
func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore {
|
||||
t.Helper()
|
||||
peer, err := New(ctx, &Config{
|
||||
Path: t.TempDir(),
|
||||
CommitterName: "test",
|
||||
CommitterEmail: "test@example.com",
|
||||
ServerHost: "127.0.0.1",
|
||||
ServerPort: port,
|
||||
Database: store.database,
|
||||
MaxOpenConns: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open store on %s through port %d: %v", store.database, port, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = peer.Close() })
|
||||
return peer
|
||||
}
|
||||
205
internal/storage/dolt/sql_wire_proxy_test.go
Normal file
205
internal/storage/dolt/sql_wire_proxy_test.go
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// comQuery is the MySQL command byte of a text-protocol statement. Every
|
||||
// statement a store sends travels as one: its DSN interpolates parameters.
|
||||
const comQuery = 0x03
|
||||
|
||||
// sqlWireProxy forwards MySQL wire traffic between stores under test and the
|
||||
// test Dolt server. It records every statement it forwards and can trap the
|
||||
// next one that matches a predicate, either holding it until the test releases
|
||||
// it (to order two writers' commits deterministically) or forwarding it and
|
||||
// cutting the client off before the server's answer arrives (a response lost
|
||||
// after the server acted on the statement). Production code needs no test hook.
|
||||
type sqlWireProxy struct {
|
||||
ln net.Listener
|
||||
target string
|
||||
|
||||
mu sync.Mutex
|
||||
trap *sqlWireTrap
|
||||
held []*sqlWireTrap
|
||||
queries []string
|
||||
conns []net.Conn
|
||||
}
|
||||
|
||||
type sqlWireTrap struct {
|
||||
match func(query string) bool
|
||||
drop bool
|
||||
sprung chan struct{}
|
||||
release chan struct{}
|
||||
released sync.Once
|
||||
}
|
||||
|
||||
func (trap *sqlWireTrap) open() { trap.released.Do(func() { close(trap.release) }) }
|
||||
|
||||
func startSQLWireProxy(t *testing.T) *sqlWireProxy {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("proxy listen: %v", err)
|
||||
}
|
||||
p := &sqlWireProxy{ln: ln, target: fmt.Sprintf("127.0.0.1:%d", testServerPort)}
|
||||
go func() {
|
||||
for {
|
||||
client, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go p.handle(client)
|
||||
}
|
||||
}()
|
||||
t.Cleanup(p.close)
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) port() int { return p.ln.Addr().(*net.TCPAddr).Port }
|
||||
|
||||
// holdNext traps the next statement matching match and keeps it from the
|
||||
// server until release is called. sprung is closed once it is caught.
|
||||
func (p *sqlWireProxy) holdNext(match func(query string) bool) (sprung <-chan struct{}, release func()) {
|
||||
trap := p.arm(match, false)
|
||||
return trap.sprung, trap.open
|
||||
}
|
||||
|
||||
// dropResponseToNext traps the next statement matching match: the server runs
|
||||
// it, and the client's connection is cut instead of receiving the answer.
|
||||
func (p *sqlWireProxy) dropResponseToNext(match func(query string) bool) (sprung <-chan struct{}) {
|
||||
return p.arm(match, true).sprung
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) arm(match func(query string) bool, drop bool) *sqlWireTrap {
|
||||
trap := &sqlWireTrap{match: match, drop: drop, sprung: make(chan struct{}), release: make(chan struct{})}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.trap != nil {
|
||||
p.trap.open()
|
||||
}
|
||||
p.trap = trap
|
||||
return trap
|
||||
}
|
||||
|
||||
// mark returns a position in the statement log for statementsSince.
|
||||
func (p *sqlWireProxy) mark() int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return len(p.queries)
|
||||
}
|
||||
|
||||
// statementsSince counts the statements forwarded after mark that match.
|
||||
func (p *sqlWireProxy) statementsSince(mark int, match func(query string) bool) int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
n := 0
|
||||
for _, q := range p.queries[mark:] {
|
||||
if match(q) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// intercept logs query and hands back the armed trap when it catches query.
|
||||
func (p *sqlWireProxy) intercept(query string) *sqlWireTrap {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.queries = append(p.queries, query)
|
||||
trap := p.trap
|
||||
if trap == nil || !trap.match(query) {
|
||||
return nil
|
||||
}
|
||||
p.trap = nil
|
||||
if !trap.drop {
|
||||
p.held = append(p.held, trap)
|
||||
}
|
||||
close(trap.sprung)
|
||||
return trap
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) close() {
|
||||
_ = p.ln.Close()
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.trap != nil {
|
||||
p.trap.open()
|
||||
}
|
||||
for _, trap := range p.held {
|
||||
trap.open()
|
||||
}
|
||||
for _, c := range p.conns {
|
||||
_ = c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) handle(client net.Conn) {
|
||||
server, err := net.Dial("tcp", p.target)
|
||||
if err != nil {
|
||||
_ = client.Close()
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.conns = append(p.conns, client, server)
|
||||
p.mu.Unlock()
|
||||
var once sync.Once
|
||||
closeBoth := func() { once.Do(func() { _ = client.Close(); _ = server.Close() }) }
|
||||
var dropping atomic.Bool
|
||||
go func() {
|
||||
buf := make([]byte, 64*1024)
|
||||
for {
|
||||
n, err := server.Read(buf)
|
||||
if n > 0 {
|
||||
if dropping.Load() {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
if _, werr := client.Write(buf[:n]); werr != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
hdr := make([]byte, 4)
|
||||
for {
|
||||
if _, err := io.ReadFull(client, hdr); err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
n := int(hdr[0]) | int(hdr[1])<<8 | int(hdr[2])<<16
|
||||
packet := make([]byte, 4+n)
|
||||
copy(packet, hdr)
|
||||
if _, err := io.ReadFull(client, packet[4:]); err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
// A command starts a new exchange (sequence id 0); the handshake's
|
||||
// packets do not.
|
||||
if hdr[3] == 0 && n > 1 && packet[4] == comQuery {
|
||||
if trap := p.intercept(strings.TrimSpace(string(packet[5:]))); trap != nil {
|
||||
if trap.drop {
|
||||
dropping.Store(true)
|
||||
} else {
|
||||
<-trap.release
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := server.Write(packet); err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isCommitStatement(query string) bool { return strings.EqualFold(query, "COMMIT") }
|
||||
|
|
@ -86,7 +86,8 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop
|
|||
msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier)
|
||||
if err := schema.DrainCall(ctx, tx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)",
|
||||
msg, s.store.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) {
|
||||
return fmt.Errorf("dolt commit: %w", err)
|
||||
// The sweep's commit point, as for the Deleter.
|
||||
return wrapSQLCommitError("dolt commit", err)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue