Compare commits
4 commits
main
...
fix/replay
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0b9e19f232 | ||
|
|
c035e8d319 | ||
|
|
2d063e575f | ||
|
|
30c3fbc137 |
25 changed files with 2070 additions and 28 deletions
|
|
@ -1231,6 +1231,7 @@ go_test(
|
|||
"todo_embedded_test.go",
|
||||
"tracker_frontdoor_proxy_test.go",
|
||||
"tracker_proxy_uow_conformance_integration_test.go",
|
||||
"transaction_replay_cgo_test.go",
|
||||
"types_builtin_test.go",
|
||||
"types_create_agreement_embedded_test.go",
|
||||
"types_create_agreement_test.go",
|
||||
|
|
|
|||
|
|
@ -174,17 +174,7 @@ normal 'bd' subcommands for interactive/read operations.`,
|
|||
if proxied {
|
||||
results, err = runBatchProxiedServer(ctx, ops, commitMsg)
|
||||
} else {
|
||||
results = make([]batchOpResult, 0, len(ops))
|
||||
err = transact(ctx, store, commitMsg, func(tx storage.Transaction) error {
|
||||
for _, op := range ops {
|
||||
res, rerr := runBatchOp(ctx, tx, op)
|
||||
if rerr != nil {
|
||||
return fmt.Errorf("line %d (%s): %w", op.line, op.raw, rerr)
|
||||
}
|
||||
results = append(results, res)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
results, err = runBatchTransaction(ctx, store, commitMsg, ops)
|
||||
}
|
||||
if err != nil {
|
||||
if jsonOutput {
|
||||
|
|
@ -222,6 +212,25 @@ func init() {
|
|||
rootCmd.AddCommand(batchCmd)
|
||||
}
|
||||
|
||||
// runBatchTransaction runs ops in one transaction and reports each op's
|
||||
// result. transact replays the transaction after a lost concurrent-write
|
||||
// conflict, so each attempt starts with no results.
|
||||
func runBatchTransaction(ctx context.Context, s storage.DoltStorage, commitMsg string, ops []batchOp) ([]batchOpResult, error) {
|
||||
var results []batchOpResult
|
||||
err := transact(ctx, s, commitMsg, func(tx storage.Transaction) error {
|
||||
results = make([]batchOpResult, 0, len(ops))
|
||||
for _, op := range ops {
|
||||
res, err := runBatchOp(ctx, tx, op)
|
||||
if err != nil {
|
||||
return fmt.Errorf("line %d (%s): %w", op.line, op.raw, err)
|
||||
}
|
||||
results = append(results, res)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return results, err
|
||||
}
|
||||
|
||||
// batchOp is one parsed command line.
|
||||
type batchOp struct {
|
||||
line int // 1-based source line number
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
publicops "github.com/steveyegge/beads/issueops"
|
||||
)
|
||||
|
|
@ -372,3 +374,53 @@ func TestBatchCmd_Registered(t *testing.T) {
|
|||
t.Error("batch missing --message flag")
|
||||
}
|
||||
}
|
||||
|
||||
// batchReplayStore runs each transaction's callback against batchReplayTx and
|
||||
// makes the first transaction lose a write conflict at commit, as it does when
|
||||
// a concurrent writer wins.
|
||||
type batchReplayStore struct {
|
||||
storage.DoltStorage
|
||||
transactions int
|
||||
}
|
||||
|
||||
func (s *batchReplayStore) RunInTransaction(_ context.Context, _ string, fn func(tx storage.Transaction) error) error {
|
||||
s.transactions++
|
||||
if err := fn(batchReplayTx{}); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.transactions == 1 {
|
||||
return storage.MarkTransactionConflict(errors.New("sql commit (regular): serialization failure"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type batchReplayTx struct{ storage.Transaction }
|
||||
|
||||
func (batchReplayTx) CloseIssue(context.Context, string, string, string, string) error { return nil }
|
||||
|
||||
func (batchReplayTx) UpdateIssue(context.Context, string, map[string]interface{}, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestRunBatchTransactionReplayReportsEachOpOnce: a batch whose transaction
|
||||
// loses a write conflict is replayed whole, and the report must describe the
|
||||
// transaction that committed, not the rolled-back attempt's ops as well.
|
||||
func TestRunBatchTransactionReplayReportsEachOpOnce(t *testing.T) {
|
||||
origExplicit := commandDidExplicitDoltCommit
|
||||
t.Cleanup(func() { commandDidExplicitDoltCommit = origExplicit })
|
||||
ops, err := parseBatchScript(strings.NewReader("close bd-1 done\nupdate bd-2 priority=1\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("parseBatchScript: %v", err)
|
||||
}
|
||||
st := &batchReplayStore{}
|
||||
results, err := runBatchTransaction(context.Background(), st, "bd: batch", ops)
|
||||
if err != nil {
|
||||
t.Fatalf("runBatchTransaction was not replayed to success: %v", err)
|
||||
}
|
||||
if st.transactions != 2 {
|
||||
t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.transactions)
|
||||
}
|
||||
if len(results) != len(ops) {
|
||||
t.Fatalf("results = %+v, want one per op (%d): the rolled-back attempt's ops were reported too", results, len(ops))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/storage/domain"
|
||||
"github.com/steveyegge/beads/internal/storage/issueops"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
|
|
@ -62,7 +63,13 @@ func createIssueWithDeps(ctx context.Context, st storage.DoltStorage, issue *typ
|
|||
commitMsg = "bd: create " + issue.ID
|
||||
}
|
||||
|
||||
// tx.CreateIssue writes what it mints back into issue, and the transaction
|
||||
// is replayed after a lost concurrent-write conflict: each attempt
|
||||
// restarts from the caller's request, or a replay would take the
|
||||
// rolled-back attempt's ID as the caller's own.
|
||||
restore := issueops.SnapshotCreateInputs([]*types.Issue{issue})
|
||||
return transactHonoringAutoCommit(ctx, st, commitMsg, func(tx storage.Transaction) error {
|
||||
restore()
|
||||
if err := tx.CreateIssue(ctx, issue, actor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
|
|
@ -17,11 +18,12 @@ type createAtomicFakeTx struct {
|
|||
created []*types.Issue
|
||||
deps []*types.Dependency
|
||||
failDep string // DependsOnID whose AddDependency fails
|
||||
mintID string // the ID an auto-minted create receives in this transaction
|
||||
}
|
||||
|
||||
func (t *createAtomicFakeTx) CreateIssue(_ context.Context, issue *types.Issue, _ string) error {
|
||||
if issue.ID == "" {
|
||||
issue.ID = "fake-1"
|
||||
issue.ID = t.mintID
|
||||
}
|
||||
cp := *issue
|
||||
t.created = append(t.created, &cp)
|
||||
|
|
@ -46,6 +48,10 @@ type createAtomicFakeStore struct {
|
|||
committedIssues []*types.Issue
|
||||
committedDeps []*types.Dependency
|
||||
failDep string
|
||||
// lostConflicts makes the first transactions lose a write conflict at
|
||||
// commit: the callback runs, nothing it wrote lands, and the failure is
|
||||
// marked as one a replay may retry.
|
||||
lostConflicts int
|
||||
}
|
||||
|
||||
func (s *createAtomicFakeStore) IsInfraTypeCtx(context.Context, types.IssueType) bool { return false }
|
||||
|
|
@ -90,11 +96,16 @@ func (s *createAtomicFakeStore) CreateIssue(_ context.Context, issue *types.Issu
|
|||
|
||||
func (s *createAtomicFakeStore) RunInTransaction(_ context.Context, _ string, fn func(tx storage.Transaction) error) error {
|
||||
s.txCount++
|
||||
tx := &createAtomicFakeTx{failDep: s.failDep}
|
||||
// Each transaction mints the next number, as counter mode does when a
|
||||
// concurrent writer has taken the previous one.
|
||||
tx := &createAtomicFakeTx{failDep: s.failDep, mintID: fmt.Sprintf("fake-%d", s.txCount)}
|
||||
if err := fn(tx); err != nil {
|
||||
// Rollback: discard everything recorded inside the transaction.
|
||||
return err
|
||||
}
|
||||
if s.txCount <= s.lostConflicts {
|
||||
return storage.MarkTransactionConflict(errors.New("sql commit (regular): serialization failure"))
|
||||
}
|
||||
s.committedIssues = append(s.committedIssues, tx.created...)
|
||||
s.committedDeps = append(s.committedDeps, tx.deps...)
|
||||
return nil
|
||||
|
|
@ -186,6 +197,36 @@ func TestCreateIssueWithDepsRollsBackOnDepFailure(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestCreateIssueWithDepsReplaysFromTheCallersRequest: a create that loses a
|
||||
// write conflict is replayed, and the replay must mint its own ID. Re-using
|
||||
// the rolled-back attempt's ID as if the caller had asked for it would take
|
||||
// the explicit-ID path and could overwrite the writer that won the conflict.
|
||||
func TestCreateIssueWithDepsReplaysFromTheCallersRequest(t *testing.T) {
|
||||
saveCreateAtomicGlobals(t)
|
||||
st := &createAtomicFakeStore{lostConflicts: 1}
|
||||
issue := &types.Issue{Title: "replayed create", Status: types.StatusOpen, IssueType: types.TypeTask}
|
||||
|
||||
if err := createIssueWithDeps(context.Background(), st, issue, "tester", createDepEdges{parentID: "fake-parent"}); err != nil {
|
||||
t.Fatalf("createIssueWithDeps was not replayed to success: %v", err)
|
||||
}
|
||||
if st.txCount != 2 {
|
||||
t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.txCount)
|
||||
}
|
||||
if len(st.committedIssues) != 1 || st.committedIssues[0].ID != "fake-2" {
|
||||
var ids []string
|
||||
for _, committed := range st.committedIssues {
|
||||
ids = append(ids, committed.ID)
|
||||
}
|
||||
t.Fatalf("committed issues = %v, want the one the replay minted (fake-2), not the rolled-back attempt's fake-1", ids)
|
||||
}
|
||||
if len(st.committedDeps) != 1 || st.committedDeps[0].IssueID != "fake-2" {
|
||||
t.Fatalf("committed deps = %+v, want the parent edge from fake-2", st.committedDeps)
|
||||
}
|
||||
if issue.ID != "fake-2" {
|
||||
t.Fatalf("caller's issue reports %q, want the committed fake-2", issue.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateIssueWithDepsEmptyEdgesUsesStoreCreate(t *testing.T) {
|
||||
saveCreateAtomicGlobals(t)
|
||||
st := &createAtomicFakeStore{}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,13 @@ import (
|
|||
// DOLT_COMMIT occurred, preventing the redundant maybeAutoCommit in
|
||||
// PersistentPostRun. Use this instead of calling store.RunInTransaction
|
||||
// directly from command handlers.
|
||||
//
|
||||
// A transaction that loses a write conflict with a concurrent writer is
|
||||
// replayed whole (storage.RunReplayableTransaction), so fn must be safe to run
|
||||
// again from the top: it touches the database only through tx and re-derives
|
||||
// every value it hands back on each run.
|
||||
func transact(ctx context.Context, s storage.DoltStorage, commitMsg string, fn func(tx storage.Transaction) error) error {
|
||||
err := s.RunInTransaction(ctx, commitMsg, fn)
|
||||
err := storage.RunReplayableTransaction(ctx, s, commitMsg, fn)
|
||||
if err == nil {
|
||||
commandDidExplicitDoltCommit = true
|
||||
}
|
||||
|
|
@ -26,7 +31,9 @@ func transact(ctx context.Context, s storage.DoltStorage, commitMsg string, fn f
|
|||
// part of command auto-commit policy. In batch/off modes the SQL transaction
|
||||
// still commits, but no Dolt version commit is created — the blank message
|
||||
// makes StageAndCommit a no-op, in embedded and SQL-server mode alike
|
||||
// (bd-4wamg: batch used to be silently inert in server mode).
|
||||
// (bd-4wamg: batch used to be silently inert in server mode). Like transact,
|
||||
// it replays a transaction that lost a concurrent-write conflict, so fn must
|
||||
// be safe to run again from the top.
|
||||
func transactHonoringAutoCommit(ctx context.Context, s storage.DoltStorage, commitMsg string, fn func(tx storage.Transaction) error) error {
|
||||
msg := commitMsg
|
||||
committedExplicitly := strings.TrimSpace(msg) != ""
|
||||
|
|
@ -39,7 +46,7 @@ func transactHonoringAutoCommit(ctx context.Context, s storage.DoltStorage, comm
|
|||
committedExplicitly = false
|
||||
}
|
||||
|
||||
err = s.RunInTransaction(ctx, msg, fn)
|
||||
err = storage.RunReplayableTransaction(ctx, s, msg, fn)
|
||||
if err == nil && committedExplicitly {
|
||||
commandDidExplicitDoltCommit = true
|
||||
}
|
||||
|
|
|
|||
183
cmd/bd/transaction_replay_cgo_test.go
Normal file
183
cmd/bd/transaction_replay_cgo_test.go
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
//go:build cgo
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/storage/dolt"
|
||||
"github.com/steveyegge/beads/internal/testutil"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// openJournalStores opens n stores (n connection pools, as n concurrent bd
|
||||
// processes would have) on one fresh database of the test server, with the
|
||||
// events journal on: every mutation also bumps the single bd_events_seq row
|
||||
// inside its transaction, so writers whose transactions overlap collide at
|
||||
// commit.
|
||||
func openJournalStores(t *testing.T, n int) []*dolt.DoltStore {
|
||||
t.Helper()
|
||||
ensureTestMode(t)
|
||||
if testDoltServerPort == 0 {
|
||||
t.Skip("Dolt test server not available, skipping")
|
||||
}
|
||||
if testutil.DoltContainerCrashed() {
|
||||
t.Skipf("Dolt test server crashed: %v", testutil.DoltContainerCrashError())
|
||||
}
|
||||
dbName := uniqueTestDBName(t)
|
||||
stores := make([]*dolt.DoltStore, 0, n)
|
||||
t.Cleanup(func() {
|
||||
for _, s := range stores {
|
||||
_ = s.Close()
|
||||
}
|
||||
dropTestDatabase(dbName, testDoltServerPort)
|
||||
})
|
||||
for i := 0; i < n; i++ {
|
||||
dbPath := filepath.Join(t.TempDir(), ".beads", "beads.db")
|
||||
writeTestMetadata(t, dbPath, dbName)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testStoreOpenTimeout)
|
||||
doltNewMutex.Lock()
|
||||
s, err := dolt.New(ctx, &dolt.Config{
|
||||
Path: dbPath,
|
||||
ServerHost: "127.0.0.1",
|
||||
ServerPort: testDoltServerPort,
|
||||
Database: dbName,
|
||||
CreateIfMissing: i == 0,
|
||||
MaxOpenConns: 4,
|
||||
})
|
||||
doltNewMutex.Unlock()
|
||||
if err == nil && i == 0 {
|
||||
err = s.SetConfig(ctx, "issue_prefix", "rp")
|
||||
}
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatalf("open store %d on %s: %v", i, dbName, err)
|
||||
}
|
||||
s.SetEventsJournalEnabled(true)
|
||||
stores = append(stores, s)
|
||||
}
|
||||
return stores
|
||||
}
|
||||
|
||||
// replayBarrierStore holds its first transaction open after the callback has
|
||||
// written, until released, so a test can order two writers' commits.
|
||||
type replayBarrierStore struct {
|
||||
*dolt.DoltStore
|
||||
attempts atomic.Int32
|
||||
prepared chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func newReplayBarrierStore(s *dolt.DoltStore) *replayBarrierStore {
|
||||
return &replayBarrierStore{DoltStore: s, prepared: make(chan struct{}), release: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (b *replayBarrierStore) RunInTransaction(ctx context.Context, msg string, fn func(tx storage.Transaction) error) error {
|
||||
return b.DoltStore.RunInTransaction(ctx, msg, func(tx storage.Transaction) error {
|
||||
if err := fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.attempts.Add(1) == 1 {
|
||||
close(b.prepared)
|
||||
select {
|
||||
case <-b.release:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// threeStepFormula is a cooked formula: a root and three chained steps, the
|
||||
// shape `bd mol wisp` clones.
|
||||
func threeStepFormula() *TemplateSubgraph {
|
||||
root := &types.Issue{ID: "formula", Title: "three-step formula", IssueType: types.TypeEpic, Status: types.StatusOpen, Priority: 2}
|
||||
sg := &TemplateSubgraph{Root: root, Issues: []*types.Issue{root}, IssueMap: map[string]*types.Issue{root.ID: root}}
|
||||
var previous *types.Issue
|
||||
for _, name := range []string{"first", "second", "third"} {
|
||||
step := &types.Issue{ID: "formula." + name, Title: name + " step", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}
|
||||
sg.Issues = append(sg.Issues, step)
|
||||
sg.IssueMap[step.ID] = step
|
||||
sg.Dependencies = append(sg.Dependencies, &types.Dependency{IssueID: step.ID, DependsOnID: root.ID, Type: types.DepParentChild})
|
||||
if previous != nil {
|
||||
sg.Dependencies = append(sg.Dependencies, &types.Dependency{IssueID: step.ID, DependsOnID: previous.ID, Type: types.DepBlocks})
|
||||
}
|
||||
previous = step
|
||||
}
|
||||
return sg
|
||||
}
|
||||
|
||||
func countWisps(t *testing.T, ctx context.Context, db *sql.DB, ids map[string]string) int {
|
||||
t.Helper()
|
||||
total := 0
|
||||
for _, id := range ids {
|
||||
var n int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM wisps WHERE id = ?", id).Scan(&n); err != nil {
|
||||
t.Fatalf("count wisp %s: %v", id, err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("wisp %s has %d rows, want exactly 1", id, n)
|
||||
}
|
||||
total += n
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// TestMolWispReplaysLostJournalConflict: two `bd mol wisp` runs instantiate a
|
||||
// formula at the same moment with the events journal on, and the second one
|
||||
// to commit loses with "sql commit (regular): Error 1213 ... try restarting
|
||||
// transaction". The clone must replay its whole transaction and succeed, with
|
||||
// every step created exactly once.
|
||||
func TestMolWispReplaysLostJournalConflict(t *testing.T) {
|
||||
stores := openJournalStores(t, 2)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
winner, loser := newReplayBarrierStore(stores[0]), newReplayBarrierStore(stores[1])
|
||||
type outcome struct {
|
||||
res *InstantiateResult
|
||||
err error
|
||||
}
|
||||
done := [2]chan outcome{make(chan outcome, 1), make(chan outcome, 1)}
|
||||
for i, s := range []*replayBarrierStore{winner, loser} {
|
||||
go func() {
|
||||
res, err := cloneSubgraph(ctx, s, threeStepFormula(), CloneOptions{
|
||||
Actor: fmt.Sprintf("writer-%d", i), Ephemeral: true, Prefix: types.IDPrefixWisp,
|
||||
})
|
||||
done[i] <- outcome{res, err}
|
||||
}()
|
||||
}
|
||||
for _, s := range []*replayBarrierStore{winner, loser} {
|
||||
select {
|
||||
case <-s.prepared:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("a clone never wrote its wisps: %v", ctx.Err())
|
||||
}
|
||||
}
|
||||
close(winner.release)
|
||||
won := <-done[0]
|
||||
close(loser.release)
|
||||
lost := <-done[1]
|
||||
|
||||
if won.err != nil {
|
||||
t.Fatalf("winner: %v", won.err)
|
||||
}
|
||||
if lost.err != nil {
|
||||
t.Fatalf("loser failed instead of replaying the conflict: %v", lost.err)
|
||||
}
|
||||
if got := loser.attempts.Load(); got != 2 {
|
||||
t.Fatalf("loser transaction attempts = %d, want 2 (one lost to the conflict, one replay)", got)
|
||||
}
|
||||
db := stores[0].DB()
|
||||
if n := countWisps(t, ctx, db, won.res.IDMapping) + countWisps(t, ctx, db, lost.res.IDMapping); n != 8 {
|
||||
t.Fatalf("wisp rows = %d, want 8 (two clones of a root and three steps)", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -62,6 +62,7 @@ go_library(
|
|||
"remote_ref.go",
|
||||
"storage.go",
|
||||
"sync.go",
|
||||
"transaction_replay.go",
|
||||
"version_control.go",
|
||||
"versioned.go",
|
||||
"wake_defers.go",
|
||||
|
|
@ -69,11 +70,16 @@ go_library(
|
|||
importpath = "github.com/steveyegge/beads/internal/storage",
|
||||
visibility = ["//:__subpackages__"],
|
||||
deps = [
|
||||
"//internal/debug",
|
||||
"//internal/hooks",
|
||||
"//internal/types",
|
||||
"//issueops",
|
||||
"//journalops",
|
||||
"//memoryops",
|
||||
"@com_github_cenkalti_backoff_v4//:backoff",
|
||||
"@io_opentelemetry_go_otel//:otel",
|
||||
"@io_opentelemetry_go_otel//attribute",
|
||||
"@io_opentelemetry_go_otel_metric//:metric",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
@ -102,6 +108,7 @@ go_test(
|
|||
"hook_role_census_test.go",
|
||||
"journal_facade_test.go",
|
||||
"leg_contract_wiring_test.go",
|
||||
"merge_slot_test.go",
|
||||
"metadata_cas_test.go",
|
||||
"metadata_edit_value_test.go",
|
||||
"metadata_jsonpath_test.go",
|
||||
|
|
@ -109,6 +116,7 @@ go_test(
|
|||
"metadata_test.go",
|
||||
"remote_ref_test.go",
|
||||
"role_accessor_decorator_test.go",
|
||||
"transaction_replay_test.go",
|
||||
],
|
||||
data = [
|
||||
":tree_go_srcs",
|
||||
|
|
@ -123,6 +131,10 @@ go_test(
|
|||
"//issueops",
|
||||
"//journalops",
|
||||
"//memoryops",
|
||||
"@com_github_cenkalti_backoff_v4//:backoff",
|
||||
"@io_opentelemetry_go_otel//attribute",
|
||||
"@io_opentelemetry_go_otel_sdk_metric//:metric",
|
||||
"@io_opentelemetry_go_otel_sdk_metric//metricdata",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ go_test(
|
|||
"create_batch_equivalence_test.go",
|
||||
"create_guard_test.go",
|
||||
"create_issue_commit_test.go",
|
||||
"create_replay_test.go",
|
||||
"create_rollback_test.go",
|
||||
"credentials_test.go",
|
||||
"cross_project_test.go",
|
||||
|
|
@ -289,6 +290,7 @@ go_test(
|
|||
"molecule_contract_test.go",
|
||||
"multistore_multiprocess_test.go",
|
||||
"open_test.go",
|
||||
"peer_store_test.go",
|
||||
"pool_timeouts_test.go",
|
||||
"port_provenance_test.go",
|
||||
"portsource_propagation_test.go",
|
||||
|
|
@ -342,6 +344,7 @@ go_test(
|
|||
"smart_gate_data_behind_test.go",
|
||||
"smart_remote_migrate_gate_test.go",
|
||||
"socket_fallback_test.go",
|
||||
"sql_wire_proxy_test.go",
|
||||
"stats_reporter_contract_test.go",
|
||||
"store_branch_pin_test.go",
|
||||
"store_unit_test.go",
|
||||
|
|
@ -352,8 +355,10 @@ go_test(
|
|||
"testmain_helper_sentinel_test.go",
|
||||
"testmain_test.go",
|
||||
"transaction_borrow_test.go",
|
||||
"transaction_conflict_test.go",
|
||||
"transaction_ignored_checkout_test.go",
|
||||
"transaction_phase_test.go",
|
||||
"transaction_replay_test.go",
|
||||
"transaction_test.go",
|
||||
"transaction_wisp_lifecycle_persistence_test.go",
|
||||
"tree_walker_contract_test.go",
|
||||
|
|
|
|||
297
internal/storage/dolt/create_replay_test.go
Normal file
297
internal/storage/dolt/create_replay_test.go
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/storage/issueops"
|
||||
"github.com/steveyegge/beads/internal/storage/rowid"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// A create that loses Dolt's commit-time merge to a concurrent writer is
|
||||
// rolled back by the server and replayed by withRetryTx. These tests prove the
|
||||
// replay starts from the caller's request: it mints its own ID and derives its
|
||||
// inline edges and comments from that ID, rather than re-using what the
|
||||
// rolled-back attempt wrote back into the caller's issue.
|
||||
|
||||
// TestConcurrentCounterModeCreatesNeverShareAnID: in counter mode two writers
|
||||
// that overlap mint the same number, so one of them loses at commit and is
|
||||
// replayed. A replay that re-used the rolled-back attempt's ID took the
|
||||
// explicit-ID upsert path and overwrote the winner's issue, and both callers
|
||||
// were told the same ID.
|
||||
func TestConcurrentCounterModeCreatesNeverShareAnID(t *testing.T) {
|
||||
base, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
if err := base.SetConfig(ctx, "issue_id_mode", "counter"); err != nil {
|
||||
t.Fatalf("set counter mode: %v", err)
|
||||
}
|
||||
const writers, rounds = 4, 3
|
||||
stores := []*DoltStore{base}
|
||||
for len(stores) < writers {
|
||||
stores = append(stores, openPeerStore(t, ctx, base))
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
titleByID := map[string]string{}
|
||||
errs := make(chan error, writers*rounds*3)
|
||||
record := func(issue *types.Issue) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if prev, dup := titleByID[issue.ID]; dup {
|
||||
errs <- fmt.Errorf("id %s handed to both %q and %q", issue.ID, prev, issue.Title)
|
||||
return
|
||||
}
|
||||
titleByID[issue.ID] = issue.Title
|
||||
}
|
||||
newIssue := func(title string) *types.Issue {
|
||||
return &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
for w, s := range stores {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
for r := range rounds {
|
||||
single := newIssue(fmt.Sprintf("single %d-%d", w, r))
|
||||
if err := s.CreateIssue(ctx, single, "writer"); err != nil {
|
||||
errs <- fmt.Errorf("create %q: %w", single.Title, err)
|
||||
} else {
|
||||
record(single)
|
||||
}
|
||||
batch := []*types.Issue{newIssue(fmt.Sprintf("batch %d-%d-a", w, r)), newIssue(fmt.Sprintf("batch %d-%d-b", w, r))}
|
||||
if err := s.CreateIssues(ctx, batch, "writer"); err != nil {
|
||||
errs <- fmt.Errorf("create batch %d-%d: %w", w, r, err)
|
||||
} else {
|
||||
record(batch[0])
|
||||
record(batch[1])
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Error(err)
|
||||
}
|
||||
for id, title := range titleByID {
|
||||
got, err := base.GetIssue(ctx, id)
|
||||
if err != nil {
|
||||
t.Errorf("read %s: %v", id, err)
|
||||
continue
|
||||
}
|
||||
if got.Title != title {
|
||||
t.Errorf("issue %s holds %q, want %q: a replayed create overwrote it", id, got.Title, title)
|
||||
}
|
||||
}
|
||||
if want := writers * rounds * 3; len(titleByID) != want && !t.Failed() {
|
||||
t.Fatalf("distinct issues = %d, want %d", len(titleByID), want)
|
||||
}
|
||||
}
|
||||
|
||||
// replayIDModes are the two ID modes a replayed create must restart in: in
|
||||
// counter mode the competitor mints the very number the loser's rolled-back
|
||||
// attempt had; in hash mode the replay mints a fresh ID, and the events journal
|
||||
// makes the two overlapping writers collide.
|
||||
var replayIDModes = []struct {
|
||||
name string
|
||||
counterMode bool
|
||||
}{
|
||||
{name: "counter mode", counterMode: true},
|
||||
{name: "hash mode", counterMode: false},
|
||||
}
|
||||
|
||||
func setUpReplayIDMode(t *testing.T, ctx context.Context, base *DoltStore, counterMode bool) {
|
||||
t.Helper()
|
||||
if counterMode {
|
||||
if err := base.SetConfig(ctx, "issue_id_mode", "counter"); err != nil {
|
||||
t.Fatalf("set counter mode: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
base.SetEventsJournalEnabled(true)
|
||||
}
|
||||
|
||||
// TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue holds a batch create
|
||||
// open (through the skipped-edge callback it runs inside its transaction) until
|
||||
// a competitor has committed, so the batch loses at commit and is replayed. Its
|
||||
// inline edge named no source, so the create filled the source in from the
|
||||
// minted ID: the replay must fill it in again from its own ID.
|
||||
func TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue(t *testing.T) {
|
||||
for _, mode := range replayIDModes {
|
||||
t.Run(mode.name, func(t *testing.T) {
|
||||
base, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
setUpReplayIDMode(t, ctx, base, mode.counterMode)
|
||||
peer := openPeerStore(t, ctx, base)
|
||||
peer.SetEventsJournalEnabled(!mode.counterMode)
|
||||
if err := base.CreateIssue(ctx, &types.Issue{ID: "test-anchor", Title: "anchor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil {
|
||||
t.Fatalf("seed anchor: %v", err)
|
||||
}
|
||||
|
||||
prepared, release := make(chan struct{}), make(chan struct{})
|
||||
var attempts atomic.Int32
|
||||
opts := storage.BatchCreateOptions{
|
||||
// Runs inside the create transaction, after the anchor edge is
|
||||
// written: hold the first attempt open until the competitor
|
||||
// has committed.
|
||||
OnSkippedDependency: func(_, _, _ string) {
|
||||
if attempts.Add(1) == 1 {
|
||||
close(prepared)
|
||||
<-release
|
||||
}
|
||||
},
|
||||
}
|
||||
loser := &types.Issue{
|
||||
Title: "loser", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask,
|
||||
Dependencies: []*types.Dependency{
|
||||
{DependsOnID: "test-anchor", Type: types.DepBlocks},
|
||||
{DependsOnID: "test-missing", Type: types.DepRelated},
|
||||
},
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- base.CreateIssuesWithFullOptions(ctx, []*types.Issue{loser}, "loser", opts) }()
|
||||
waitForTransactionPrepared(t, ctx, prepared, "loser")
|
||||
competitor := &types.Issue{Title: "competitor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := peer.CreateIssue(ctx, competitor, "competitor"); err != nil {
|
||||
close(release)
|
||||
t.Fatalf("competitor create: %v", err)
|
||||
}
|
||||
close(release)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("loser create was not replayed to success: %v", err)
|
||||
}
|
||||
if n := attempts.Load(); n < 2 {
|
||||
t.Fatalf("loser create ran %d attempt(s), want a replay: the competitor's commit did not conflict", n)
|
||||
}
|
||||
assertCompetitorIntact(t, ctx, base, loser, competitor)
|
||||
if got := edgeSourcesInto(t, ctx, base, "test-anchor"); len(got) != 1 || got[0] != loser.ID {
|
||||
t.Fatalf("edges into test-anchor come from %v, want exactly [%s] (the loser's own issue); competitor is %s", got, loser.ID, competitor.ID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue holds a single
|
||||
// create's COMMIT at a proxy until a competitor has committed, so the create
|
||||
// loses and is replayed. Its inline comment carried no ID, so the create
|
||||
// derived one from the minted issue ID: the replay must derive it again from
|
||||
// its own.
|
||||
func TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue(t *testing.T) {
|
||||
for _, mode := range replayIDModes {
|
||||
t.Run(mode.name, func(t *testing.T) {
|
||||
base, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
setUpReplayIDMode(t, ctx, base, mode.counterMode)
|
||||
proxy := startSQLWireProxy(t)
|
||||
via := openStoreOnPort(t, ctx, base, proxy.port())
|
||||
via.SetEventsJournalEnabled(!mode.counterMode)
|
||||
|
||||
loser := &types.Issue{
|
||||
Title: "loser", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask,
|
||||
Comments: []*types.Comment{{Author: "loser", Text: "filed with the issue"}},
|
||||
}
|
||||
mark := proxy.mark()
|
||||
held, release := proxy.holdNext(isCommitStatement)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- via.CreateIssue(ctx, loser, "loser") }()
|
||||
select {
|
||||
case <-held:
|
||||
case err := <-done:
|
||||
t.Fatalf("loser create finished before its COMMIT was held: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("loser create never reached COMMIT: %v", ctx.Err())
|
||||
}
|
||||
competitor := &types.Issue{Title: "competitor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := base.CreateIssue(ctx, competitor, "competitor"); err != nil {
|
||||
release()
|
||||
t.Fatalf("competitor create: %v", err)
|
||||
}
|
||||
release()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("loser create was not replayed to success: %v", err)
|
||||
}
|
||||
insertsComment := func(q string) bool { return strings.HasPrefix(q, "INSERT INTO comments") }
|
||||
if n := proxy.statementsSince(mark, insertsComment); n != 2 {
|
||||
t.Fatalf("loser create wrote its comment in %d attempt(s), want 2 (one lost to the competitor, one replay)", n)
|
||||
}
|
||||
assertCompetitorIntact(t, ctx, base, loser, competitor)
|
||||
assertCommentDerivedFromOwnIssue(t, ctx, base, loser.ID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertCompetitorIntact(t *testing.T, ctx context.Context, base *DoltStore, loser, competitor *types.Issue) {
|
||||
t.Helper()
|
||||
if loser.ID == competitor.ID {
|
||||
t.Fatalf("loser and competitor were both handed %s", loser.ID)
|
||||
}
|
||||
got, err := base.GetIssue(ctx, competitor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("read competitor %s: %v", competitor.ID, err)
|
||||
}
|
||||
if got.Title != "competitor" {
|
||||
t.Fatalf("competitor %s holds %q: the replayed create overwrote it", competitor.ID, got.Title)
|
||||
}
|
||||
}
|
||||
|
||||
func edgeSourcesInto(t *testing.T, ctx context.Context, s *DoltStore, target string) []string {
|
||||
t.Helper()
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT issue_id FROM dependencies WHERE depends_on_issue_id = ?", target)
|
||||
if err != nil {
|
||||
t.Fatalf("query edges into %s: %v", target, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var sources []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
t.Fatalf("scan edge source: %v", err)
|
||||
}
|
||||
sources = append(sources, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("read edges into %s: %v", target, err)
|
||||
}
|
||||
sort.Strings(sources)
|
||||
return sources
|
||||
}
|
||||
|
||||
// assertCommentDerivedFromOwnIssue checks issueID's one comment carries the ID
|
||||
// derived from issueID itself, not one left over from a rolled-back attempt
|
||||
// that derived it from that attempt's issue ID.
|
||||
func assertCommentDerivedFromOwnIssue(t *testing.T, ctx context.Context, s *DoltStore, issueID string) {
|
||||
t.Helper()
|
||||
comments, err := s.GetIssueComments(ctx, issueID)
|
||||
if err != nil {
|
||||
t.Fatalf("read comments of %s: %v", issueID, err)
|
||||
}
|
||||
if len(comments) != 1 {
|
||||
t.Fatalf("%s has %d comments, want 1", issueID, len(comments))
|
||||
}
|
||||
c := comments[0]
|
||||
want := rowid.New("comments", 0, rowid.Digest([]sql.NullString{
|
||||
{String: issueID, Valid: true}, {String: c.Author, Valid: true},
|
||||
{String: c.Text, Valid: true}, {String: issueops.FormatAuxTime(c.CreatedAt), Valid: true},
|
||||
}))
|
||||
if c.ID != want {
|
||||
t.Fatalf("comment on %s has id %s, want %s (derived from its own issue): the replay re-used the rolled-back attempt's comment id", issueID, c.ID, want)
|
||||
}
|
||||
}
|
||||
|
|
@ -40,7 +40,14 @@ func (s *DoltStore) createIssue(ctx context.Context, issue *types.Issue, actor s
|
|||
}
|
||||
|
||||
var result issueops.CreateIssueResult
|
||||
// withRetryTx replays the body after a lost commit-time conflict, and the
|
||||
// create writes its minted ID (and what derives from it) back into the
|
||||
// caller's issue: each attempt restarts from the caller's request, or the
|
||||
// replay would take the rolled-back ID as the caller's own and upsert over
|
||||
// the winner's row.
|
||||
restore := issueops.SnapshotCreateInputs([]*types.Issue{issue})
|
||||
if err := s.withRetryTx(ctx, func(tx *sql.Tx) error {
|
||||
restore()
|
||||
// SkipPrefixValidation matches legacy behavior: single-issue path does
|
||||
// not validate prefixes for explicit IDs.
|
||||
bc, err := issueops.NewBatchContext(ctx, tx, storage.BatchCreateOptions{
|
||||
|
|
@ -113,14 +120,19 @@ func (s *DoltStore) createIssuesWithFullOptions(ctx context.Context, issues []*t
|
|||
issue.Ephemeral = true
|
||||
}
|
||||
}
|
||||
restore := issueops.SnapshotCreateInputs(issues)
|
||||
return s.withRetryTx(ctx, func(tx *sql.Tx) error {
|
||||
restore()
|
||||
_, err := issueops.CreateIssuesInTxWithResult(ctx, tx, issues, actor, opts)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// Each attempt restarts from the caller's request; see createIssue.
|
||||
restore := issueops.SnapshotCreateInputs(issues)
|
||||
var result issueops.CreateIssuesResult
|
||||
if err := s.withRetryTx(ctx, func(tx *sql.Tx) error {
|
||||
restore()
|
||||
var err error
|
||||
result, err = issueops.CreateIssuesInTxWithResult(ctx, tx, issues, actor, opts)
|
||||
return err
|
||||
|
|
|
|||
33
internal/storage/dolt/peer_store_test.go
Normal file
33
internal/storage/dolt/peer_store_test.go
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// openPeerStore opens a second store on store's database, with a connection
|
||||
// pool of its own as a second bd process would have.
|
||||
func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore {
|
||||
t.Helper()
|
||||
return openStoreOnPort(t, ctx, store, testServerPort)
|
||||
}
|
||||
|
||||
// openStoreOnPort opens a store on store's database through whatever listens
|
||||
// on port: the test server itself, or a test proxy in front of it.
|
||||
func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore {
|
||||
t.Helper()
|
||||
peer, err := New(ctx, &Config{
|
||||
Path: t.TempDir(),
|
||||
CommitterName: "test",
|
||||
CommitterEmail: "test@example.com",
|
||||
ServerHost: "127.0.0.1",
|
||||
ServerPort: port,
|
||||
Database: store.database,
|
||||
MaxOpenConns: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open store on %s through port %d: %v", store.database, port, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = peer.Close() })
|
||||
return peer
|
||||
}
|
||||
205
internal/storage/dolt/sql_wire_proxy_test.go
Normal file
205
internal/storage/dolt/sql_wire_proxy_test.go
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// comQuery is the MySQL command byte of a text-protocol statement. Every
|
||||
// statement a store sends travels as one: its DSN interpolates parameters.
|
||||
const comQuery = 0x03
|
||||
|
||||
// sqlWireProxy forwards MySQL wire traffic between stores under test and the
|
||||
// test Dolt server. It records every statement it forwards and can trap the
|
||||
// next one that matches a predicate, either holding it until the test releases
|
||||
// it (to order two writers' commits deterministically) or forwarding it and
|
||||
// cutting the client off before the server's answer arrives (a response lost
|
||||
// after the server acted on the statement). Production code needs no test hook.
|
||||
type sqlWireProxy struct {
|
||||
ln net.Listener
|
||||
target string
|
||||
|
||||
mu sync.Mutex
|
||||
trap *sqlWireTrap
|
||||
held []*sqlWireTrap
|
||||
queries []string
|
||||
conns []net.Conn
|
||||
}
|
||||
|
||||
type sqlWireTrap struct {
|
||||
match func(query string) bool
|
||||
drop bool
|
||||
sprung chan struct{}
|
||||
release chan struct{}
|
||||
released sync.Once
|
||||
}
|
||||
|
||||
func (trap *sqlWireTrap) open() { trap.released.Do(func() { close(trap.release) }) }
|
||||
|
||||
func startSQLWireProxy(t *testing.T) *sqlWireProxy {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("proxy listen: %v", err)
|
||||
}
|
||||
p := &sqlWireProxy{ln: ln, target: fmt.Sprintf("127.0.0.1:%d", testServerPort)}
|
||||
go func() {
|
||||
for {
|
||||
client, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go p.handle(client)
|
||||
}
|
||||
}()
|
||||
t.Cleanup(p.close)
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) port() int { return p.ln.Addr().(*net.TCPAddr).Port }
|
||||
|
||||
// holdNext traps the next statement matching match and keeps it from the
|
||||
// server until release is called. sprung is closed once it is caught.
|
||||
func (p *sqlWireProxy) holdNext(match func(query string) bool) (sprung <-chan struct{}, release func()) {
|
||||
trap := p.arm(match, false)
|
||||
return trap.sprung, trap.open
|
||||
}
|
||||
|
||||
// dropResponseToNext traps the next statement matching match: the server runs
|
||||
// it, and the client's connection is cut instead of receiving the answer.
|
||||
func (p *sqlWireProxy) dropResponseToNext(match func(query string) bool) (sprung <-chan struct{}) {
|
||||
return p.arm(match, true).sprung
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) arm(match func(query string) bool, drop bool) *sqlWireTrap {
|
||||
trap := &sqlWireTrap{match: match, drop: drop, sprung: make(chan struct{}), release: make(chan struct{})}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.trap != nil {
|
||||
p.trap.open()
|
||||
}
|
||||
p.trap = trap
|
||||
return trap
|
||||
}
|
||||
|
||||
// mark returns a position in the statement log for statementsSince.
|
||||
func (p *sqlWireProxy) mark() int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return len(p.queries)
|
||||
}
|
||||
|
||||
// statementsSince counts the statements forwarded after mark that match.
|
||||
func (p *sqlWireProxy) statementsSince(mark int, match func(query string) bool) int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
n := 0
|
||||
for _, q := range p.queries[mark:] {
|
||||
if match(q) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// intercept logs query and hands back the armed trap when it catches query.
|
||||
func (p *sqlWireProxy) intercept(query string) *sqlWireTrap {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.queries = append(p.queries, query)
|
||||
trap := p.trap
|
||||
if trap == nil || !trap.match(query) {
|
||||
return nil
|
||||
}
|
||||
p.trap = nil
|
||||
if !trap.drop {
|
||||
p.held = append(p.held, trap)
|
||||
}
|
||||
close(trap.sprung)
|
||||
return trap
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) close() {
|
||||
_ = p.ln.Close()
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.trap != nil {
|
||||
p.trap.open()
|
||||
}
|
||||
for _, trap := range p.held {
|
||||
trap.open()
|
||||
}
|
||||
for _, c := range p.conns {
|
||||
_ = c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *sqlWireProxy) handle(client net.Conn) {
|
||||
server, err := net.Dial("tcp", p.target)
|
||||
if err != nil {
|
||||
_ = client.Close()
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.conns = append(p.conns, client, server)
|
||||
p.mu.Unlock()
|
||||
var once sync.Once
|
||||
closeBoth := func() { once.Do(func() { _ = client.Close(); _ = server.Close() }) }
|
||||
var dropping atomic.Bool
|
||||
go func() {
|
||||
buf := make([]byte, 64*1024)
|
||||
for {
|
||||
n, err := server.Read(buf)
|
||||
if n > 0 {
|
||||
if dropping.Load() {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
if _, werr := client.Write(buf[:n]); werr != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
hdr := make([]byte, 4)
|
||||
for {
|
||||
if _, err := io.ReadFull(client, hdr); err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
n := int(hdr[0]) | int(hdr[1])<<8 | int(hdr[2])<<16
|
||||
packet := make([]byte, 4+n)
|
||||
copy(packet, hdr)
|
||||
if _, err := io.ReadFull(client, packet[4:]); err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
// A command starts a new exchange (sequence id 0); the handshake's
|
||||
// packets do not.
|
||||
if hdr[3] == 0 && n > 1 && packet[4] == comQuery {
|
||||
if trap := p.intercept(strings.TrimSpace(string(packet[5:]))); trap != nil {
|
||||
if trap.drop {
|
||||
dropping.Store(true)
|
||||
} else {
|
||||
<-trap.release
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := server.Write(packet); err != nil {
|
||||
closeBoth()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isCommitStatement(query string) bool { return strings.EqualFold(query, "COMMIT") }
|
||||
|
|
@ -67,10 +67,12 @@ func (t *doltTransaction) CreateIssueImport(ctx context.Context, issue *types.Is
|
|||
|
||||
// RunInTransaction executes a function within a database transaction. Its
|
||||
// callback is invoked at most once per call; callers retry explicitly after a
|
||||
// callback has started when their operation is safe to repeat. The commitMsg is
|
||||
// used for the DOLT_COMMIT that makes regular writes visible in Dolt history.
|
||||
// Wisp routing is handled by individual transaction methods based on
|
||||
// ID/Ephemeral.
|
||||
// callback has started when their operation is safe to repeat. A failure that
|
||||
// proves the server rolled the whole transaction back after losing a write
|
||||
// conflict is marked storage.ErrTransactionConflict so such a caller can tell
|
||||
// (storage.RunReplayableTransaction). The commitMsg is used for the
|
||||
// DOLT_COMMIT that makes regular writes visible in Dolt history. Wisp routing
|
||||
// is handled by individual transaction methods based on ID/Ephemeral.
|
||||
func (s *DoltStore) RunInTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error {
|
||||
return s.runInTransaction(ctx, commitMsg, fn, s.runDoltTransaction)
|
||||
}
|
||||
|
|
@ -98,12 +100,37 @@ func (s *DoltStore) runInTransaction(
|
|||
if callbackErr == nil && errors.Is(err, ErrCommitIndeterminate) {
|
||||
return err
|
||||
}
|
||||
return backoff.Permanent(err)
|
||||
return backoff.Permanent(markTransactionConflict(err))
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// markTransactionConflict tags a failure that proves the server rolled the
|
||||
// whole transaction back after losing a write conflict as
|
||||
// storage.ErrTransactionConflict, leaving its message unchanged. Nothing such a
|
||||
// transaction wrote is durable, so a caller whose callback is safe to repeat
|
||||
// may run it again.
|
||||
//
|
||||
// - 1213 (serialization failure or deadlock) is what Dolt's commit-time merge
|
||||
// raises when a concurrent writer committed a conflicting change first;
|
||||
// the remedy is exactly a replay.
|
||||
// - 1205 (lock-wait timeout) is tagged for engines that raise it. The engine
|
||||
// may roll back only the statement, but the failed transaction is always
|
||||
// rolled back whole here before any replay.
|
||||
//
|
||||
// Dolt's 1105 "@autocommit transaction rolled back" is not tagged: it reports
|
||||
// merge conflicts already in the working set, which a replay meets again. A
|
||||
// failure after the SQL commit landed carries ErrCommitIndeterminate and is
|
||||
// never tagged: its rows may be durable, and replaying them could apply them
|
||||
// twice.
|
||||
func markTransactionConflict(err error) error {
|
||||
if err == nil || errors.Is(err, ErrCommitIndeterminate) || !isSerializationError(err) {
|
||||
return err
|
||||
}
|
||||
return storage.MarkTransactionConflict(err)
|
||||
}
|
||||
|
||||
// RunInIssueLifecycleTransaction runs a lifecycle transition and its durable
|
||||
// side effects through one SQL transaction and one Dolt commit attempt.
|
||||
func (s *DoltStore) RunInIssueLifecycleTransaction(ctx context.Context, commitMsg string, fn func(tx storage.IssueLifecycleTransaction) error) error {
|
||||
|
|
@ -140,7 +167,7 @@ func (s *DoltStore) runInIssueLifecycleTransaction(
|
|||
if callbackErr == nil && errors.Is(err, ErrCommitIndeterminate) {
|
||||
return err
|
||||
}
|
||||
return backoff.Permanent(err)
|
||||
return backoff.Permanent(markTransactionConflict(err))
|
||||
}
|
||||
return err
|
||||
})
|
||||
|
|
|
|||
119
internal/storage/dolt/transaction_conflict_test.go
Normal file
119
internal/storage/dolt/transaction_conflict_test.go
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
mysql "github.com/go-sql-driver/mysql"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
)
|
||||
|
||||
// transactionFailureCases are failures a public transaction can surface after
|
||||
// its callback ran, and whether each proves the server rolled the whole
|
||||
// transaction back after losing a write conflict, so that a caller may replay
|
||||
// it.
|
||||
func transactionFailureCases() []struct {
|
||||
name string
|
||||
err error
|
||||
conflict bool
|
||||
} {
|
||||
serialization := &mysql.MySQLError{Number: 1213, SQLState: [5]byte{'4', '0', '0', '0', '1'}, Message: "serialization failure: this transaction conflicts with a committed transaction from another client, try restarting transaction"}
|
||||
return []struct {
|
||||
name string
|
||||
err error
|
||||
conflict bool
|
||||
}{
|
||||
{name: "1213 serialization failure at the regular commit", err: wrapSQLCommitError("sql commit (regular)", serialization), conflict: true},
|
||||
{name: "1205 lock wait timeout", err: fmt.Errorf("callback write: %w", &mysql.MySQLError{Number: 1205, Message: "Lock wait timeout exceeded; try restarting transaction"}), conflict: true},
|
||||
// Dolt's autocommit rollback reports merge conflicts already in the
|
||||
// working set; a replay meets them again, so it is not a conflict a
|
||||
// replay can win.
|
||||
{name: "Dolt autocommit rollback", err: &mysql.MySQLError{Number: 1105, Message: "Merge conflict detected, @autocommit transaction rolled back"}, conflict: false},
|
||||
{name: "other 1105", err: &mysql.MySQLError{Number: 1105, Message: "command denied to user 'op'@'%'"}, conflict: false},
|
||||
{name: "duplicate key", err: &mysql.MySQLError{Number: 1062, Message: "Duplicate entry"}, conflict: false},
|
||||
// A serialization failure on the Dolt commit that follows a
|
||||
// successful SQL commit does not roll the transaction back: its rows
|
||||
// landed.
|
||||
{name: "1213 after the SQL commit landed", err: fmt.Errorf("stage and commit after regular SQL commit: %w: %w", serialization, ErrCommitIndeterminate), conflict: false},
|
||||
{name: "lost commit response", err: wrapSQLCommitError("sql commit (regular)", mysql.ErrInvalidConn), conflict: false},
|
||||
// Classification is by the decoded server error, never by text.
|
||||
{name: "untyped text naming 1213", err: errors.New("Error 1213 (40001): serialization failure"), conflict: false},
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunInTransactionMarksOnlyRollbackGuaranteedConflicts(t *testing.T) {
|
||||
for _, tc := range transactionFailureCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := &DoltStore{}
|
||||
callbacks := 0
|
||||
err := store.runInTransaction(context.Background(), "test: conflict classification", func(storage.Transaction) error {
|
||||
callbacks++
|
||||
return nil
|
||||
}, func(_ context.Context, _ string, fn func(storage.Transaction) error) error {
|
||||
if err := fn(nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return tc.err
|
||||
})
|
||||
assertTransactionConflictClassification(t, err, tc.err, tc.conflict)
|
||||
if callbacks != 1 {
|
||||
t.Fatalf("callback calls = %d, want 1: RunInTransaction itself never replays a callback", callbacks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunInIssueLifecycleTransactionMarksOnlyRollbackGuaranteedConflicts(t *testing.T) {
|
||||
for _, tc := range transactionFailureCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := &DoltStore{}
|
||||
callbacks := 0
|
||||
err := store.runInIssueLifecycleTransaction(context.Background(), "test: conflict classification", func(storage.IssueLifecycleTransaction) error {
|
||||
callbacks++
|
||||
return nil
|
||||
}, func(_ context.Context, fn func(*sql.Tx) error) error {
|
||||
if err := fn(nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return tc.err
|
||||
})
|
||||
assertTransactionConflictClassification(t, err, tc.err, tc.conflict)
|
||||
if callbacks != 1 {
|
||||
t.Fatalf("callback calls = %d, want 1", callbacks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertTransactionConflictClassification(t *testing.T, err, cause error, conflict bool) {
|
||||
t.Helper()
|
||||
if !errors.Is(err, cause) {
|
||||
t.Fatalf("err = %v, want the original failure %v in the chain", err, cause)
|
||||
}
|
||||
if got := errors.Is(err, storage.ErrTransactionConflict); got != conflict {
|
||||
t.Fatalf("errors.Is(err, ErrTransactionConflict) = %v, want %v (err = %v)", got, conflict, err)
|
||||
}
|
||||
if err.Error() != cause.Error() {
|
||||
t.Fatalf("message = %q, want the unchanged %q", err.Error(), cause.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// A callback's own failure that wraps a rollback-guaranteed conflict (a write
|
||||
// statement inside the transaction that lost) is a conflict too: the callback
|
||||
// error rolls the transaction back.
|
||||
func TestRunInTransactionMarksConflictRaisedInsideCallback(t *testing.T) {
|
||||
store := &DoltStore{}
|
||||
inner := &mysql.MySQLError{Number: 1213, Message: "serialization failure"}
|
||||
err := store.runInTransaction(context.Background(), "test: conflict in callback", func(storage.Transaction) error {
|
||||
return fmt.Errorf("adding dependency: %w", inner)
|
||||
}, func(_ context.Context, _ string, fn func(storage.Transaction) error) error {
|
||||
return fn(nil)
|
||||
})
|
||||
if !errors.Is(err, storage.ErrTransactionConflict) || !errors.Is(err, inner) {
|
||||
t.Fatalf("err = %v, want a marked conflict wrapping the statement's 1213", err)
|
||||
}
|
||||
}
|
||||
329
internal/storage/dolt/transaction_replay_test.go
Normal file
329
internal/storage/dolt/transaction_replay_test.go
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// With the events journal on, every mutation also bumps the single
|
||||
// bd_events_seq row inside its own transaction, so two writers whose
|
||||
// transactions overlap at all collide when the second one commits: Dolt rolls
|
||||
// it back with MySQL 1213 ("serialization failure ... try restarting
|
||||
// transaction"). These tests order two such transactions deterministically:
|
||||
// both write, the winner commits, then the loser commits and loses.
|
||||
|
||||
// overlapCreates runs two transactions that each create one issue, holds both
|
||||
// open until both have written, then lets the winner commit before the loser.
|
||||
// run decides how each transaction is driven; the hold applies only to each
|
||||
// transaction's first attempt.
|
||||
func overlapCreates(t *testing.T, ctx context.Context, winner, loser *DoltStore, ephemeral bool,
|
||||
run func(s *DoltStore, fn func(tx storage.Transaction) error) error,
|
||||
) (winnerErr, loserErr error, loserCallbacks int32, ids [2]string) {
|
||||
t.Helper()
|
||||
prepared := [2]chan struct{}{make(chan struct{}), make(chan struct{})}
|
||||
release := [2]chan struct{}{make(chan struct{}), make(chan struct{})}
|
||||
done := [2]chan error{make(chan error, 1), make(chan error, 1)}
|
||||
var calls [2]atomic.Int32
|
||||
for i, s := range []*DoltStore{winner, loser} {
|
||||
go func() {
|
||||
done[i] <- run(s, func(tx storage.Transaction) error {
|
||||
attempt := calls[i].Add(1)
|
||||
issue := &types.Issue{
|
||||
Title: []string{"winner", "loser"}[i],
|
||||
Status: types.StatusOpen,
|
||||
Priority: 2,
|
||||
IssueType: types.TypeTask,
|
||||
Ephemeral: ephemeral,
|
||||
}
|
||||
if err := tx.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
return err
|
||||
}
|
||||
ids[i] = issue.ID
|
||||
if attempt == 1 {
|
||||
close(prepared[i])
|
||||
return waitForTransactionRelease(ctx, release[i])
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
}
|
||||
waitForTransactionPrepared(t, ctx, prepared[0], "winner")
|
||||
waitForTransactionPrepared(t, ctx, prepared[1], "loser")
|
||||
close(release[0])
|
||||
winnerErr = <-done[0]
|
||||
close(release[1])
|
||||
loserErr = <-done[1]
|
||||
return winnerErr, loserErr, calls[1].Load(), ids
|
||||
}
|
||||
|
||||
func countRows(t *testing.T, ctx context.Context, db *sql.DB, table, column, value string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
//nolint:gosec // G202: table and column are fixed literals chosen by the test
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table+" WHERE "+column+" = ?", value).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s rows with %s = %s: %v", table, column, value, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func journalRows(t *testing.T, ctx context.Context, db *sql.DB) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal").Scan(&n); err != nil {
|
||||
t.Fatalf("count journal rows: %v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestRunInTransactionMarksLostJournalConflict: RunInTransaction keeps its
|
||||
// at-most-once contract and does not replay the loser, but marks its failure
|
||||
// as a rolled-back conflict so a caller whose callback is safe to repeat can.
|
||||
func TestRunInTransactionMarksLostJournalConflict(t *testing.T) {
|
||||
store, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
peer := openPeerStore(t, ctx, store)
|
||||
store.SetEventsJournalEnabled(true)
|
||||
peer.SetEventsJournalEnabled(true)
|
||||
|
||||
winnerErr, loserErr, loserCalls, _ := overlapCreates(t, ctx, store, peer, true,
|
||||
func(s *DoltStore, fn func(tx storage.Transaction) error) error {
|
||||
return s.RunInTransaction(ctx, "bd: clone template subgraph", fn)
|
||||
})
|
||||
if winnerErr != nil {
|
||||
t.Fatalf("winner: %v", winnerErr)
|
||||
}
|
||||
if loserErr == nil {
|
||||
t.Fatal("loser committed although both transactions bumped the journal counter; the overlap did not conflict")
|
||||
}
|
||||
if !errors.Is(loserErr, storage.ErrTransactionConflict) {
|
||||
t.Fatalf("loser error = %v, want it marked storage.ErrTransactionConflict", loserErr)
|
||||
}
|
||||
if errors.Is(loserErr, ErrCommitIndeterminate) {
|
||||
t.Fatalf("loser error = %v must not be indeterminate: the server rolled it back", loserErr)
|
||||
}
|
||||
if loserCalls != 1 {
|
||||
t.Fatalf("loser callback ran %d times, want 1 (RunInTransaction never replays)", loserCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunReplayableTransactionReplaysLostJournalConflict drives the same
|
||||
// overlap through storage.RunReplayableTransaction: the loser's WHOLE
|
||||
// transaction is replayed once, both creates land exactly once, and the
|
||||
// journal records exactly one create for each.
|
||||
func TestRunReplayableTransactionReplaysLostJournalConflict(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ephemeral bool
|
||||
table string
|
||||
}{
|
||||
{name: "wisp", ephemeral: true, table: "wisps"},
|
||||
{name: "durable", ephemeral: false, table: "issues"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
peer := openPeerStore(t, ctx, store)
|
||||
store.SetEventsJournalEnabled(true)
|
||||
peer.SetEventsJournalEnabled(true)
|
||||
journalBefore := journalRows(t, ctx, store.db)
|
||||
|
||||
winnerErr, loserErr, loserCalls, ids := overlapCreates(t, ctx, store, peer, tc.ephemeral,
|
||||
func(s *DoltStore, fn func(tx storage.Transaction) error) error {
|
||||
return storage.RunReplayableTransaction(ctx, s, "bd: clone template subgraph", fn)
|
||||
})
|
||||
if winnerErr != nil {
|
||||
t.Fatalf("winner: %v", winnerErr)
|
||||
}
|
||||
if loserErr != nil {
|
||||
t.Fatalf("loser was not replayed to success: %v", loserErr)
|
||||
}
|
||||
if loserCalls != 2 {
|
||||
t.Fatalf("loser callback ran %d times, want 2 (one lost attempt, one replay)", loserCalls)
|
||||
}
|
||||
for i, id := range ids {
|
||||
if id == "" {
|
||||
t.Fatalf("transaction %d reported no issue id", i)
|
||||
}
|
||||
if n := countRows(t, ctx, store.db, tc.table, "id", id); n != 1 {
|
||||
t.Fatalf("%s rows for %s = %d, want exactly 1", tc.table, id, n)
|
||||
}
|
||||
}
|
||||
perCreate := countRows(t, ctx, store.db, "bd_events_journal", "issue_id", ids[0])
|
||||
if grew := journalRows(t, ctx, store.db) - journalBefore; grew != 2*perCreate {
|
||||
t.Fatalf("journal grew by %d rows, want %d: the replay must record its create exactly as the winner did", grew, 2*perCreate)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunReplayableTransactionNeverReplaysLostCommitResponse: a COMMIT whose
|
||||
// response is lost may have landed, so it surfaces as indeterminate and is
|
||||
// never replayed, with the journal on (one transaction) or off (a regular and
|
||||
// an ignored transaction).
|
||||
func TestRunReplayableTransactionNeverReplaysLostCommitResponse(t *testing.T) {
|
||||
base, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
proxy := startSQLWireProxy(t)
|
||||
via := openStoreOnPort(t, ctx, base, proxy.port())
|
||||
|
||||
for _, journal := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("journal=%v", journal), func(t *testing.T) {
|
||||
via.SetEventsJournalEnabled(journal)
|
||||
title := fmt.Sprintf("lost-commit-journal-%v", journal)
|
||||
var calls atomic.Int32
|
||||
sprung := proxy.dropResponseToNext(isCommitStatement)
|
||||
err := storage.RunReplayableTransaction(ctx, via, "bd: lost commit", func(tx storage.Transaction) error {
|
||||
calls.Add(1)
|
||||
return tx.CreateIssue(ctx, &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "tester")
|
||||
})
|
||||
select {
|
||||
case <-sprung:
|
||||
default:
|
||||
t.Fatalf("the transaction never reached COMMIT (err = %v)", err)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("a lost commit response reported success")
|
||||
}
|
||||
if !errors.Is(err, ErrCommitIndeterminate) || errors.Is(err, storage.ErrTransactionConflict) {
|
||||
t.Fatalf("err = %v, want indeterminate and not a replayable conflict", err)
|
||||
}
|
||||
if n := calls.Load(); n != 1 {
|
||||
t.Fatalf("callback ran %d times: a lost commit response must never be replayed", n)
|
||||
}
|
||||
if n := countRows(t, ctx, base.db, "issues", "title", title); n != 1 {
|
||||
t.Fatalf("rows titled %q = %d, want 1: the server committed it once", title, n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// splitTxOverlap runs a winner on peer and a loser on store, both with the
|
||||
// journal off, so each runs the default split shape: a regular transaction
|
||||
// for Dolt-tracked tables and an ignored one for the wisp tables. Both write,
|
||||
// the winner commits, then the loser commits.
|
||||
func splitTxOverlap(t *testing.T, ctx context.Context, store, peer *DoltStore,
|
||||
winnerWrite, loserWrite func(tx storage.Transaction) error,
|
||||
) (loserErr error, loserCalls int32) {
|
||||
t.Helper()
|
||||
prepared := [2]chan struct{}{make(chan struct{}), make(chan struct{})}
|
||||
release := [2]chan struct{}{make(chan struct{}), make(chan struct{})}
|
||||
done := [2]chan error{make(chan error, 1), make(chan error, 1)}
|
||||
var calls atomic.Int32
|
||||
go func() {
|
||||
done[0] <- peer.RunInTransaction(ctx, "winner", func(tx storage.Transaction) error {
|
||||
if err := winnerWrite(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
close(prepared[0])
|
||||
return waitForTransactionRelease(ctx, release[0])
|
||||
})
|
||||
}()
|
||||
go func() {
|
||||
done[1] <- storage.RunReplayableTransaction(ctx, store, "loser", func(tx storage.Transaction) error {
|
||||
attempt := calls.Add(1)
|
||||
if err := loserWrite(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if attempt == 1 {
|
||||
close(prepared[1])
|
||||
return waitForTransactionRelease(ctx, release[1])
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
waitForTransactionPrepared(t, ctx, prepared[0], "winner")
|
||||
waitForTransactionPrepared(t, ctx, prepared[1], "loser")
|
||||
close(release[0])
|
||||
if err := <-done[0]; err != nil {
|
||||
t.Fatalf("winner: %v", err)
|
||||
}
|
||||
close(release[1])
|
||||
return <-done[1], calls.Load()
|
||||
}
|
||||
|
||||
// TestSplitTransactionRegularConflictRollsBackBothPlanes: the loser's REGULAR
|
||||
// commit loses, so both of its transactions roll back and the replay writes
|
||||
// each plane exactly once; no wisp from the lost attempt survives.
|
||||
func TestSplitTransactionRegularConflictRollsBackBothPlanes(t *testing.T) {
|
||||
store, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
peer := openPeerStore(t, ctx, store)
|
||||
createPerm(t, ctx, store, "test-shared")
|
||||
|
||||
loserErr, loserCalls := splitTxOverlap(t, ctx, store, peer,
|
||||
func(tx storage.Transaction) error {
|
||||
return tx.UpdateIssue(ctx, "test-shared", map[string]interface{}{"title": "winner"}, "w")
|
||||
},
|
||||
func(tx storage.Transaction) error {
|
||||
if err := tx.UpdateIssue(ctx, "test-shared", map[string]interface{}{"title": "loser"}, "l"); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.CreateIssue(ctx, &types.Issue{Title: "split-wisp", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Ephemeral: true}, "l")
|
||||
})
|
||||
if loserErr != nil {
|
||||
t.Fatalf("loser: %v", loserErr)
|
||||
}
|
||||
if loserCalls != 2 {
|
||||
t.Fatalf("loser attempts = %d, want 2", loserCalls)
|
||||
}
|
||||
if n := countRows(t, ctx, store.db, "wisps", "title", "split-wisp"); n != 1 {
|
||||
t.Fatalf("wisp rows = %d, want 1: the lost attempt's ignored-plane write leaked", n)
|
||||
}
|
||||
got, err := store.GetIssue(ctx, "test-shared")
|
||||
if err != nil {
|
||||
t.Fatalf("read test-shared: %v", err)
|
||||
}
|
||||
if got.Title != "loser" {
|
||||
t.Fatalf("title = %q, want the replayed loser's", got.Title)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSplitTransactionIgnoredConflictAfterRegularCommitIsNotReplayed: the
|
||||
// loser's IGNORED commit loses after its regular commit landed. That outcome
|
||||
// is partial, so it is indeterminate and never a replayable conflict.
|
||||
func TestSplitTransactionIgnoredConflictAfterRegularCommitIsNotReplayed(t *testing.T) {
|
||||
store, cleanup := setupConcurrentTestStore(t)
|
||||
defer cleanup()
|
||||
ctx, cancel := concurrentTestContext(t)
|
||||
defer cancel()
|
||||
peer := openPeerStore(t, ctx, store)
|
||||
createWisp(t, ctx, store, "test-wisp-shared")
|
||||
|
||||
loserErr, loserCalls := splitTxOverlap(t, ctx, store, peer,
|
||||
func(tx storage.Transaction) error {
|
||||
return tx.UpdateIssue(ctx, "test-wisp-shared", map[string]interface{}{"title": "winner"}, "w")
|
||||
},
|
||||
func(tx storage.Transaction) error {
|
||||
if err := tx.CreateIssue(ctx, &types.Issue{Title: "split-durable", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "l"); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.UpdateIssue(ctx, "test-wisp-shared", map[string]interface{}{"title": "loser"}, "l")
|
||||
})
|
||||
if loserErr == nil {
|
||||
t.Fatal("loser succeeded; the ignored-plane overlap did not conflict")
|
||||
}
|
||||
if errors.Is(loserErr, storage.ErrTransactionConflict) || !errors.Is(loserErr, ErrCommitIndeterminate) {
|
||||
t.Fatalf("err = %v, want indeterminate, not a replayable conflict", loserErr)
|
||||
}
|
||||
if loserCalls != 1 {
|
||||
t.Fatalf("loser attempts = %d, want 1: no replay after the regular commit landed", loserCalls)
|
||||
}
|
||||
if n := countRows(t, ctx, store.db, "issues", "title", "split-durable"); n != 1 {
|
||||
t.Fatalf("durable rows = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -243,6 +243,9 @@ func TestRunInTransactionSerializationConflictInvokesCallbacksOnce(t *testing.T)
|
|||
if errors.Is(err, ErrCommitIndeterminate) {
|
||||
t.Fatalf("stale transaction error = %v unexpectedly marks an indeterminate commit", err)
|
||||
}
|
||||
if !errors.Is(err, storage.ErrTransactionConflict) {
|
||||
t.Fatalf("stale transaction error = %v, want it marked storage.ErrTransactionConflict so a caller may replay it", err)
|
||||
}
|
||||
if got := callsA.Load(); got != 1 {
|
||||
t.Errorf("winner callback calls = %d, want 1", got)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ go_library(
|
|||
"create_dep_batch.go",
|
||||
"create_fastpath_testhook.go",
|
||||
"create_only_guard.go",
|
||||
"create_replay.go",
|
||||
"cycle_report.go",
|
||||
"cycles.go",
|
||||
"delete.go",
|
||||
|
|
@ -145,6 +146,7 @@ go_test(
|
|||
"create_batch_cache_test.go",
|
||||
"create_only_errors_test.go",
|
||||
"create_only_guard_test.go",
|
||||
"create_replay_test.go",
|
||||
"create_test.go",
|
||||
"custom_types_parity_test.go",
|
||||
"cycle_report_test.go",
|
||||
|
|
|
|||
67
internal/storage/issueops/create_replay.go
Normal file
67
internal/storage/issueops/create_replay.go
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package issueops
|
||||
|
||||
import "github.com/steveyegge/beads/internal/types"
|
||||
|
||||
// SnapshotCreateInputs records what the create path writes back into the
|
||||
// caller's issues and returns a func that puts it back, so a create that a
|
||||
// transaction retry replays starts every attempt from the caller's request
|
||||
// instead of from what a rolled-back attempt minted.
|
||||
//
|
||||
// The create path writes back three things:
|
||||
//
|
||||
// - the Issue struct itself: the minted ID (assignCreateIssueIDInTx) and the
|
||||
// defaulted timestamps and content hash (PrepareIssueForInsert);
|
||||
// - through each inline dependency, the owning issue's ID as the edge's
|
||||
// source when the edge named none (PersistDependenciesWithOptionsResult);
|
||||
// - through each inline comment, the content-derived ID when the comment
|
||||
// carried none (PersistComments).
|
||||
//
|
||||
// Restoring only the struct would leave the last two pointing at the
|
||||
// rolled-back attempt's ID. The dependency and comment objects stay the
|
||||
// caller's: only the fields the create path writes are reset. A new write into
|
||||
// caller-owned state on the create path needs its restore here.
|
||||
func SnapshotCreateInputs(issues []*types.Issue) (restore func()) {
|
||||
type depSource struct {
|
||||
dep *types.Dependency
|
||||
issueID string
|
||||
}
|
||||
type commentID struct {
|
||||
comment *types.Comment
|
||||
id string
|
||||
}
|
||||
type issueState struct {
|
||||
issue *types.Issue
|
||||
fields types.Issue
|
||||
deps []depSource
|
||||
comments []commentID
|
||||
}
|
||||
saved := make([]issueState, 0, len(issues))
|
||||
for _, issue := range issues {
|
||||
if issue == nil {
|
||||
continue
|
||||
}
|
||||
state := issueState{issue: issue, fields: *issue}
|
||||
for _, dep := range issue.Dependencies {
|
||||
if dep != nil {
|
||||
state.deps = append(state.deps, depSource{dep: dep, issueID: dep.IssueID})
|
||||
}
|
||||
}
|
||||
for _, comment := range issue.Comments {
|
||||
if comment != nil {
|
||||
state.comments = append(state.comments, commentID{comment: comment, id: comment.ID})
|
||||
}
|
||||
}
|
||||
saved = append(saved, state)
|
||||
}
|
||||
return func() {
|
||||
for _, state := range saved {
|
||||
*state.issue = state.fields
|
||||
for _, d := range state.deps {
|
||||
d.dep.IssueID = d.issueID
|
||||
}
|
||||
for _, c := range state.comments {
|
||||
c.comment.ID = c.id
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
70
internal/storage/issueops/create_replay_test.go
Normal file
70
internal/storage/issueops/create_replay_test.go
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
package issueops
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack covers every
|
||||
// write the create path makes into a caller's issues, including the two it
|
||||
// makes through pointers the Issue struct only holds: the minted ID copied into
|
||||
// each inline dependency that named no source, and the derived ID stamped onto
|
||||
// each inline comment that had none. A struct copy restores neither.
|
||||
func TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack(t *testing.T) {
|
||||
presetEdge := &types.Dependency{IssueID: "test-explicit", DependsOnID: "test-other", Type: types.DepRelated}
|
||||
inlineEdge := &types.Dependency{DependsOnID: "test-anchor", Type: types.DepBlocks}
|
||||
inlineComment := &types.Comment{Author: "a", Text: "first"}
|
||||
importedComment := &types.Comment{ID: "c-imported", Author: "b", Text: "second"}
|
||||
issue := &types.Issue{
|
||||
Title: "loser",
|
||||
Labels: []string{"area:storage"},
|
||||
Dependencies: []*types.Dependency{inlineEdge, nil, presetEdge},
|
||||
Comments: []*types.Comment{inlineComment, importedComment, nil},
|
||||
}
|
||||
second := &types.Issue{Title: "second"}
|
||||
restore := SnapshotCreateInputs([]*types.Issue{issue, nil, second})
|
||||
|
||||
// What one create attempt leaves behind.
|
||||
issue.ID = "test-1"
|
||||
issue.CreatedAt = time.Now()
|
||||
issue.UpdatedAt = issue.CreatedAt
|
||||
issue.ContentHash = "attempt-1-hash"
|
||||
inlineEdge.IssueID = issue.ID
|
||||
inlineComment.ID = "c-attempt-1"
|
||||
second.ID = "test-2"
|
||||
|
||||
restore()
|
||||
|
||||
if issue.ID != "" || !issue.CreatedAt.IsZero() || !issue.UpdatedAt.IsZero() || issue.ContentHash != "" {
|
||||
t.Fatalf("issue after restore = id %q, created %v, updated %v, hash %q; want the caller's empty values",
|
||||
issue.ID, issue.CreatedAt, issue.UpdatedAt, issue.ContentHash)
|
||||
}
|
||||
if second.ID != "" {
|
||||
t.Fatalf("second issue id after restore = %q, want empty", second.ID)
|
||||
}
|
||||
if inlineEdge.IssueID != "" {
|
||||
t.Fatalf("inline edge source after restore = %q, want empty so the replay derives it from the new id", inlineEdge.IssueID)
|
||||
}
|
||||
if presetEdge.IssueID != "test-explicit" {
|
||||
t.Fatalf("explicit edge source after restore = %q, want the caller's %q", presetEdge.IssueID, "test-explicit")
|
||||
}
|
||||
if inlineComment.ID != "" {
|
||||
t.Fatalf("inline comment id after restore = %q, want empty so the replay derives it again", inlineComment.ID)
|
||||
}
|
||||
if importedComment.ID != "c-imported" {
|
||||
t.Fatalf("imported comment id after restore = %q, want the caller's %q", importedComment.ID, "c-imported")
|
||||
}
|
||||
if issue.Dependencies[0] != inlineEdge || issue.Comments[0] != inlineComment {
|
||||
t.Fatal("restore replaced the caller's dependency or comment objects instead of resetting them")
|
||||
}
|
||||
|
||||
// Every replay starts from the same request.
|
||||
issue.ID = "test-3"
|
||||
inlineEdge.IssueID = issue.ID
|
||||
restore()
|
||||
if issue.ID != "" || inlineEdge.IssueID != "" {
|
||||
t.Fatalf("second restore left id %q and edge source %q", issue.ID, inlineEdge.IssueID)
|
||||
}
|
||||
}
|
||||
|
|
@ -78,8 +78,10 @@ func MergeSlotCheckImpl(ctx context.Context, s Storage) (*MergeSlotStatus, error
|
|||
}
|
||||
|
||||
// MergeSlotAcquireImpl is the shared implementation of Storage.MergeSlotAcquire.
|
||||
// It uses RunInTransaction to ensure atomic check-and-set, preventing two
|
||||
// agents from simultaneously acquiring the slot.
|
||||
// It runs the check-and-set in one transaction, preventing two agents from
|
||||
// simultaneously acquiring the slot. Two agents acquiring at once is exactly
|
||||
// the race the slot arbitrates: the loser's transaction is replayed whole, so
|
||||
// it re-reads the slot and queues behind the winner instead of failing.
|
||||
func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, wait bool) (*MergeSlotResult, error) {
|
||||
if holder == "" {
|
||||
return nil, fmt.Errorf("merge-slot acquire: holder must not be empty")
|
||||
|
|
@ -87,11 +89,11 @@ func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string,
|
|||
|
||||
slotID := MergeSlotID(ctx, s)
|
||||
var result MergeSlotResult
|
||||
result.SlotID = slotID
|
||||
|
||||
err := s.RunInTransaction(ctx,
|
||||
err := RunReplayableTransaction(ctx, s,
|
||||
fmt.Sprintf("bd: acquire merge slot %s for %s", slotID, holder),
|
||||
func(tx Transaction) error {
|
||||
result = MergeSlotResult{SlotID: slotID}
|
||||
slot, err := tx.GetIssue(ctx, slotID)
|
||||
if err != nil || slot == nil {
|
||||
return fmt.Errorf("merge slot not found: %s (run 'bd merge-slot create' first)", slotID)
|
||||
|
|
@ -153,7 +155,7 @@ func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string,
|
|||
func MergeSlotReleaseImpl(ctx context.Context, s Storage, holder, actor string) error {
|
||||
slotID := MergeSlotID(ctx, s)
|
||||
|
||||
return s.RunInTransaction(ctx,
|
||||
return RunReplayableTransaction(ctx, s,
|
||||
fmt.Sprintf("bd: release merge slot %s", slotID),
|
||||
func(tx Transaction) error {
|
||||
slot, err := tx.GetIssue(ctx, slotID)
|
||||
|
|
|
|||
83
internal/storage/merge_slot_test.go
Normal file
83
internal/storage/merge_slot_test.go
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// mergeSlotRaceStore holds one merge slot row and runs each transaction's
|
||||
// callback against a copy of it, keeping the callback's writes only when the
|
||||
// transaction commits. Its first transaction loses a write conflict to a
|
||||
// competitor whose acquire commits first.
|
||||
type mergeSlotRaceStore struct {
|
||||
Storage
|
||||
slot types.Issue
|
||||
competitor string
|
||||
transactions int
|
||||
}
|
||||
|
||||
func (s *mergeSlotRaceStore) GetConfig(context.Context, string) (string, error) { return "test", nil }
|
||||
|
||||
func (s *mergeSlotRaceStore) RunInTransaction(_ context.Context, _ string, fn func(tx Transaction) error) error {
|
||||
s.transactions++
|
||||
tx := &mergeSlotRaceTx{slot: s.slot}
|
||||
if err := fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.transactions == 1 {
|
||||
s.slot.Status = types.StatusInProgress
|
||||
s.slot.Metadata = json.RawMessage(`{"holder":"` + s.competitor + `"}`)
|
||||
return MarkTransactionConflict(errors.New("sql commit (regular): serialization failure"))
|
||||
}
|
||||
s.slot = tx.slot
|
||||
return nil
|
||||
}
|
||||
|
||||
type mergeSlotRaceTx struct {
|
||||
Transaction
|
||||
slot types.Issue
|
||||
}
|
||||
|
||||
func (tx *mergeSlotRaceTx) GetIssue(_ context.Context, id string) (*types.Issue, error) {
|
||||
if id != tx.slot.ID {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
slot := tx.slot
|
||||
return &slot, nil
|
||||
}
|
||||
|
||||
func (tx *mergeSlotRaceTx) UpdateIssue(_ context.Context, _ string, updates map[string]interface{}, _ string) error {
|
||||
if status, ok := updates["status"].(types.Status); ok {
|
||||
tx.slot.Status = status
|
||||
}
|
||||
if meta, ok := updates["metadata"].(string); ok {
|
||||
tx.slot.Metadata = json.RawMessage(meta)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestMergeSlotAcquireReplayReportsTheReplaysOutcome: two agents acquiring at
|
||||
// once is the race the slot arbitrates. The loser's transaction is replayed,
|
||||
// finds the slot held, and must report exactly that, not the acquisition its
|
||||
// rolled-back attempt made.
|
||||
func TestMergeSlotAcquireReplayReportsTheReplaysOutcome(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
st := &mergeSlotRaceStore{slot: types.Issue{ID: "test-merge-slot", Status: types.StatusOpen}, competitor: "other"}
|
||||
res, err := MergeSlotAcquireImpl(context.Background(), st, "me", "me", false)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire was not replayed to an answer: %v", err)
|
||||
}
|
||||
if st.transactions != 2 {
|
||||
t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.transactions)
|
||||
}
|
||||
if res.Acquired || res.Holder != "other" || res.SlotID != "test-merge-slot" {
|
||||
t.Fatalf("result = %+v, want the slot reported held by other: the rolled-back attempt's acquisition leaked into the replay's result", res)
|
||||
}
|
||||
if st.slot.Status != types.StatusInProgress || string(st.slot.Metadata) != `{"holder":"other"}` {
|
||||
t.Fatalf("slot = %s %s, want the competitor's acquisition intact", st.slot.Status, st.slot.Metadata)
|
||||
}
|
||||
}
|
||||
|
|
@ -636,7 +636,10 @@ type Storage interface {
|
|||
// starts, it is invoked at most once for this public call: callers retry
|
||||
// explicitly when repeating their work is safe. An error wrapping
|
||||
// ErrCommitIndeterminate means the durable outcome may be unknown and must
|
||||
// not be blindly replayed.
|
||||
// not be blindly replayed. An error wrapping ErrTransactionConflict means
|
||||
// the transaction lost a concurrent-write conflict and was rolled back
|
||||
// whole; RunReplayableTransaction replays such a transaction for a caller
|
||||
// whose callback is safe to repeat.
|
||||
RunInTransaction(ctx context.Context, commitMsg string, fn func(tx Transaction) error) error
|
||||
|
||||
// MergeSlot — serialized conflict resolution primitive.
|
||||
|
|
|
|||
160
internal/storage/transaction_replay.go
Normal file
160
internal/storage/transaction_replay.go
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
|
||||
"github.com/steveyegge/beads/internal/debug"
|
||||
)
|
||||
|
||||
// ErrTransactionConflict marks a transaction the database rolled back whole
|
||||
// because it lost a write conflict with a concurrent transaction: a
|
||||
// serialization failure or deadlock (MySQL 1213, what Dolt's commit-time merge
|
||||
// raises) or a lock-wait timeout (1205). Nothing the transaction wrote is
|
||||
// durable, and the server's own remedy is to run the WHOLE transaction again.
|
||||
// Backends attach it with MarkTransactionConflict; it never accompanies
|
||||
// ErrCommitIndeterminate.
|
||||
var ErrTransactionConflict = errors.New("transaction lost a write conflict with a concurrent transaction and was rolled back")
|
||||
|
||||
// transactionConflictError tags a backend error as ErrTransactionConflict
|
||||
// without changing the message the user sees.
|
||||
type transactionConflictError struct{ err error }
|
||||
|
||||
func (e *transactionConflictError) Error() string { return e.err.Error() }
|
||||
|
||||
func (e *transactionConflictError) Unwrap() []error {
|
||||
return []error{e.err, ErrTransactionConflict}
|
||||
}
|
||||
|
||||
// MarkTransactionConflict returns err tagged so that
|
||||
// errors.Is(err, ErrTransactionConflict) holds, keeping its message and its
|
||||
// cause chain intact. Only a backend that knows the failure rolled the whole
|
||||
// transaction back may call it.
|
||||
func MarkTransactionConflict(err error) error {
|
||||
if err == nil || errors.Is(err, ErrTransactionConflict) {
|
||||
return err
|
||||
}
|
||||
return &transactionConflictError{err: err}
|
||||
}
|
||||
|
||||
// Transactor is the one store capability RunReplayableTransaction needs.
|
||||
type Transactor interface {
|
||||
RunInTransaction(ctx context.Context, commitMsg string, fn func(tx Transaction) error) error
|
||||
}
|
||||
|
||||
// The replay policy. Each attempt is a complete transaction, so the attempt
|
||||
// bound is what caps the work; the elapsed bound only stops a run whose
|
||||
// attempts are themselves slow. The waits grow exponentially, are capped so
|
||||
// the later attempts keep arriving, and are randomized by half their length in
|
||||
// either direction: writers that collide tend to start together, and a fixed
|
||||
// schedule would have every loser collide again in lockstep.
|
||||
const (
|
||||
replayMaxAttempts = 20
|
||||
replayInitialInterval = 25 * time.Millisecond
|
||||
replayMaxInterval = time.Second
|
||||
replayRandomizationFactor = 0.5
|
||||
replayMaxElapsed = 30 * time.Second
|
||||
)
|
||||
|
||||
// replayTimer supplies the timer for one replay loop; nil uses the backoff
|
||||
// library's real timer. Tests substitute one that fires immediately.
|
||||
var replayTimer func() backoff.Timer
|
||||
|
||||
// txReplays counts whole-transaction replays, labeled outcome=replayed for each
|
||||
// replay and outcome=exhausted when a loop runs out of attempts on conflicts:
|
||||
// the RunReplayableTransaction counterpart of withRetryTx's write-retry
|
||||
// counter. It is registered against the global delegating provider, so it
|
||||
// forwards to the real provider once telemetry is initialized.
|
||||
var txReplays = newTxReplaysCounter(otel.Meter("github.com/steveyegge/beads/storage"))
|
||||
|
||||
func newTxReplaysCounter(meter metric.Meter) metric.Int64Counter {
|
||||
counter, _ := meter.Int64Counter("bd.tx_replays_total",
|
||||
metric.WithDescription("Whole-transaction replays after a lost write conflict (label: outcome=replayed|exhausted)"),
|
||||
metric.WithUnit("{replay}"),
|
||||
)
|
||||
return counter
|
||||
}
|
||||
|
||||
// RunReplayableTransaction runs fn through s.RunInTransaction and, while an
|
||||
// attempt fails with ErrTransactionConflict, waits a jittered backoff and runs
|
||||
// the WHOLE transaction again: a fresh BEGIN, the callback from the top, and a
|
||||
// fresh COMMIT. RunInTransaction invokes its callback at most once and leaves
|
||||
// the decision to replay to the caller, because only the caller knows whether
|
||||
// its callback is safe to repeat.
|
||||
//
|
||||
// Use it only for a callback that touches the database exclusively through tx,
|
||||
// re-derives every value it hands back on each invocation (plain assignment or
|
||||
// an explicit reset at its top, never an append into captured state), and has
|
||||
// no other side effects. Any other failure — a callback error, a non-conflict
|
||||
// database error, or an ErrCommitIndeterminate outcome that may already have
|
||||
// landed — is returned after the one attempt that produced it.
|
||||
//
|
||||
// The loop makes at most replayMaxAttempts attempts within replayMaxElapsed.
|
||||
// When it gives up it returns the last conflict, which still matches
|
||||
// ErrTransactionConflict, annotated with the number of attempts made.
|
||||
func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg string, fn func(tx Transaction) error) error {
|
||||
bo := backoff.NewExponentialBackOff()
|
||||
bo.InitialInterval = replayInitialInterval
|
||||
bo.MaxInterval = replayMaxInterval
|
||||
bo.RandomizationFactor = replayRandomizationFactor
|
||||
bo.MaxElapsedTime = replayMaxElapsed
|
||||
policy := backoff.WithContext(backoff.WithMaxRetries(bo, replayMaxAttempts-1), ctx)
|
||||
|
||||
attempts := 0
|
||||
var lastConflict, lastAttempt error
|
||||
notify := func(err error, wait time.Duration) {
|
||||
txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed")))
|
||||
debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v\n", commitMsg, attempts, wait, err)
|
||||
}
|
||||
var timer backoff.Timer
|
||||
if replayTimer != nil {
|
||||
timer = replayTimer()
|
||||
}
|
||||
err := backoff.RetryNotifyWithTimer(func() error {
|
||||
attempts++
|
||||
err := s.RunInTransaction(ctx, commitMsg, fn)
|
||||
lastAttempt = err
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if replayableConflict(err) {
|
||||
lastConflict = err
|
||||
return err
|
||||
}
|
||||
return backoff.Permanent(err)
|
||||
}, policy, notify, timer)
|
||||
if err == nil || lastConflict == nil || !replayableConflict(lastAttempt) {
|
||||
// The last attempt's own outcome is the answer, unchanged: a callback
|
||||
// error, a non-conflict database error, or an indeterminate commit
|
||||
// that may have landed — even when the context ended during it.
|
||||
// Replacing it with an earlier, rolled-back conflict would tell the
|
||||
// caller nothing was written.
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, lastConflict) {
|
||||
// The loop ended on the conflict it was replaying: its budget ran out.
|
||||
txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "exhausted")))
|
||||
return fmt.Errorf("%w (gave up after %d attempts: the transaction kept losing write conflicts with concurrent writers)", err, attempts)
|
||||
}
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
// The context ended while waiting to replay: report both why the loop
|
||||
// stopped and the conflict it was retrying. That is not exhaustion.
|
||||
return fmt.Errorf("%w (stopped retrying after %d attempt(s): %w)", lastConflict, attempts, ctxErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// replayableConflict reports whether err proves the attempt rolled back
|
||||
// because of a concurrent-write conflict. Indeterminacy always wins: a write
|
||||
// whose commit response was lost may have landed, so it is never replayed even
|
||||
// if something also marked it as a conflict.
|
||||
func replayableConflict(err error) bool {
|
||||
return errors.Is(err, ErrTransactionConflict) && !errors.Is(err, ErrCommitIndeterminate)
|
||||
}
|
||||
313
internal/storage/transaction_replay_test.go
Normal file
313
internal/storage/transaction_replay_test.go
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
||||
)
|
||||
|
||||
// scriptedTransactor answers each RunInTransaction call with the next scripted
|
||||
// outcome, invoking the callback first as a real store would.
|
||||
type scriptedTransactor struct {
|
||||
outcomes []error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *scriptedTransactor) RunInTransaction(_ context.Context, _ string, fn func(tx Transaction) error) error {
|
||||
s.calls++
|
||||
if err := fn(nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(s.outcomes) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := s.outcomes[0]
|
||||
s.outcomes = s.outcomes[1:]
|
||||
return out
|
||||
}
|
||||
|
||||
// instantTimer fires immediately and records every wait the policy asked for,
|
||||
// so the replay loop runs without sleeping.
|
||||
type instantTimer struct {
|
||||
c chan time.Time
|
||||
delays []time.Duration
|
||||
}
|
||||
|
||||
func (t *instantTimer) Start(d time.Duration) {
|
||||
t.delays = append(t.delays, d)
|
||||
t.c <- time.Now()
|
||||
}
|
||||
func (t *instantTimer) Stop() {}
|
||||
func (t *instantTimer) C() <-chan time.Time { return t.c }
|
||||
|
||||
func useInstantReplayTimer(t *testing.T) *instantTimer {
|
||||
t.Helper()
|
||||
timer := &instantTimer{c: make(chan time.Time, 1)}
|
||||
prev := replayTimer
|
||||
replayTimer = func() backoff.Timer { return timer }
|
||||
t.Cleanup(func() { replayTimer = prev })
|
||||
return timer
|
||||
}
|
||||
|
||||
// conflict is what a backend returns when the server rolled the whole
|
||||
// transaction back after losing a concurrent-write race.
|
||||
func conflict(n int) error {
|
||||
return MarkTransactionConflict(fmt.Errorf("sql commit (regular): serialization failure #%d", n))
|
||||
}
|
||||
|
||||
func conflicts(n int) []error {
|
||||
out := make([]error, n)
|
||||
for i := range out {
|
||||
out[i] = conflict(i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestMarkTransactionConflictKeepsMessageAndCause(t *testing.T) {
|
||||
cause := errors.New("sql commit (regular): Error 1213 (40001): serialization failure")
|
||||
marked := MarkTransactionConflict(cause)
|
||||
if !errors.Is(marked, ErrTransactionConflict) {
|
||||
t.Fatalf("marked error %v does not match ErrTransactionConflict", marked)
|
||||
}
|
||||
if !errors.Is(marked, cause) {
|
||||
t.Fatalf("marked error %v lost its cause", marked)
|
||||
}
|
||||
if marked.Error() != cause.Error() {
|
||||
t.Fatalf("marked message = %q, want the unchanged %q", marked.Error(), cause.Error())
|
||||
}
|
||||
if MarkTransactionConflict(nil) != nil {
|
||||
t.Fatal("MarkTransactionConflict(nil) must stay nil")
|
||||
}
|
||||
if again := MarkTransactionConflict(marked); again != marked {
|
||||
t.Fatal("marking an already-marked error must not wrap it twice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplayableTransactionReplaysWholeTransactionOnConflict(t *testing.T) {
|
||||
timer := useInstantReplayTimer(t)
|
||||
tr := &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}}
|
||||
runs := 0
|
||||
err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error {
|
||||
runs++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunReplayableTransaction = %v, want success after two lost conflicts", err)
|
||||
}
|
||||
if tr.calls != 3 || runs != 3 {
|
||||
t.Fatalf("transactions=%d callbacks=%d, want the WHOLE transaction, callback included, run 3 times", tr.calls, runs)
|
||||
}
|
||||
if len(timer.delays) != 2 {
|
||||
t.Fatalf("backoff waits = %v, want one wait before each replay", timer.delays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplayableTransactionDoesNotReplayOtherFailures(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
for name, failure := range map[string]error{
|
||||
"plain error": errors.New("dependency would create a cycle"),
|
||||
"indeterminate commit": fmt.Errorf("stage and commit after regular SQL commit: %w", ErrCommitIndeterminate),
|
||||
// A conflict marker never overrides indeterminacy: the write may
|
||||
// already have landed, so replaying it could apply it twice.
|
||||
"marked but indeterminate": MarkTransactionConflict(fmt.Errorf("lost response: %w", ErrCommitIndeterminate)),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tr := &scriptedTransactor{outcomes: []error{failure, nil}}
|
||||
err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil })
|
||||
if !errors.Is(err, failure) {
|
||||
t.Fatalf("err = %v, want the original failure", err)
|
||||
}
|
||||
if tr.calls != 1 {
|
||||
t.Fatalf("transactions = %d, want exactly one attempt for a failure that is not a conflict", tr.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplayableTransactionCallbackErrorIsNotReplayed(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
tr := &scriptedTransactor{}
|
||||
want := errors.New("issue not found")
|
||||
runs := 0
|
||||
err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error {
|
||||
runs++
|
||||
return want
|
||||
})
|
||||
if !errors.Is(err, want) || runs != 1 {
|
||||
t.Fatalf("err=%v runs=%d, want the callback's own error after one run", err, runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplayableTransactionBoundsAttempts(t *testing.T) {
|
||||
timer := useInstantReplayTimer(t)
|
||||
tr := &scriptedTransactor{outcomes: conflicts(replayMaxAttempts + 5)}
|
||||
err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil })
|
||||
if err == nil {
|
||||
t.Fatal("RunReplayableTransaction succeeded while every attempt lost a conflict")
|
||||
}
|
||||
if tr.calls != replayMaxAttempts {
|
||||
t.Fatalf("transactions = %d, want exactly replayMaxAttempts (%d)", tr.calls, replayMaxAttempts)
|
||||
}
|
||||
if !errors.Is(err, ErrTransactionConflict) {
|
||||
t.Fatalf("exhausted error %v must still classify as a transaction conflict", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), fmt.Sprintf("serialization failure #%d", replayMaxAttempts-1)) {
|
||||
t.Fatalf("exhausted error %q must carry the LAST attempt's failure", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), fmt.Sprintf("%d attempts", replayMaxAttempts)) {
|
||||
t.Fatalf("exhausted error %q must say how many attempts were made", err)
|
||||
}
|
||||
longest := time.Duration(float64(replayMaxInterval) * (1 + replayRandomizationFactor))
|
||||
for i, d := range timer.delays {
|
||||
if d <= 0 || d > longest {
|
||||
t.Fatalf("wait %d = %v, want a positive wait no longer than the jittered cap %v", i, d, longest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplayableTransactionBackoffIsJittered(t *testing.T) {
|
||||
// Writers that collide tend to retry together; a fixed schedule would
|
||||
// have every loser collide again at the same instant.
|
||||
distinct := map[time.Duration]bool{}
|
||||
for run := 0; run < 8; run++ {
|
||||
timer := useInstantReplayTimer(t)
|
||||
tr := &scriptedTransactor{outcomes: []error{conflict(1), nil}}
|
||||
if err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(timer.delays) != 1 {
|
||||
t.Fatalf("waits = %v, want one", timer.delays)
|
||||
}
|
||||
distinct[timer.delays[0]] = true
|
||||
}
|
||||
if len(distinct) < 2 {
|
||||
t.Fatalf("first waits were identical across runs (%v); the backoff is not jittered", distinct)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplayableTransactionStopsWhenContextEnds(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
tr := &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}}
|
||||
runs := 0
|
||||
err := RunReplayableTransaction(ctx, tr, "bd: test", func(Transaction) error {
|
||||
runs++
|
||||
cancel()
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("RunReplayableTransaction kept going after its context was canceled")
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrTransactionConflict) {
|
||||
t.Fatalf("err = %v, want both the cancellation and the conflict that was being retried", err)
|
||||
}
|
||||
if runs != 1 {
|
||||
t.Fatalf("callback runs = %d, want 1 (no replay after cancellation)", runs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunReplayableTransactionCountsReplays makes replays observable the way
|
||||
// withRetryTx's retries are: each replay and each loop that gives up is
|
||||
// counted on bd.tx_replays_total, labeled by outcome. A loop that stops on a
|
||||
// different failure after a replay did not give up on a conflict.
|
||||
func TestRunReplayableTransactionCountsReplays(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
// The counter is swapped for one built by the same constructor on a
|
||||
// test-local provider, so the global meter provider is never touched.
|
||||
reader := sdkmetric.NewManualReader()
|
||||
provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
|
||||
prev := txReplays
|
||||
txReplays = newTxReplaysCounter(provider.Meter("test"))
|
||||
t.Cleanup(func() {
|
||||
txReplays = prev
|
||||
_ = provider.Shutdown(context.Background())
|
||||
})
|
||||
|
||||
noop := func(Transaction) error { return nil }
|
||||
if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}}, "bd: replayed", noop); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: conflicts(replayMaxAttempts)}, "bd: exhausted", noop); err == nil {
|
||||
t.Fatal("the exhausted loop succeeded")
|
||||
}
|
||||
if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), errors.New("constraint violated")}}, "bd: failed", noop); err == nil {
|
||||
t.Fatal("the loop that ended on a plain failure succeeded")
|
||||
}
|
||||
// The context ends while the loop waits to replay: not exhausted.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
if err := RunReplayableTransaction(ctx, &scriptedTransactor{outcomes: []error{conflict(1), nil}}, "bd: canceled", func(Transaction) error {
|
||||
cancel()
|
||||
return nil
|
||||
}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v, want the cancellation", err)
|
||||
}
|
||||
|
||||
var rm metricdata.ResourceMetrics
|
||||
if err := reader.Collect(context.Background(), &rm); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]int64{}
|
||||
for _, scope := range rm.ScopeMetrics {
|
||||
for _, m := range scope.Metrics {
|
||||
if m.Name != "bd.tx_replays_total" {
|
||||
continue
|
||||
}
|
||||
sum, ok := m.Data.(metricdata.Sum[int64])
|
||||
if !ok {
|
||||
t.Fatalf("bd.tx_replays_total is %T, want an int64 sum", m.Data)
|
||||
}
|
||||
for _, dp := range sum.DataPoints {
|
||||
outcome, _ := dp.Attributes.Value(attribute.Key("outcome"))
|
||||
got[outcome.AsString()] += dp.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
if want := int64(2 + replayMaxAttempts - 1 + 1); got["replayed"] != want {
|
||||
t.Fatalf("replayed count = %d, want %d (two, then every replay of the exhausted loop, then one)", got["replayed"], want)
|
||||
}
|
||||
if got["exhausted"] != 1 {
|
||||
t.Fatalf("exhausted count = %d, want 1 (only the loop that ran out of attempts)", got["exhausted"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome: once a replay
|
||||
// has run, the loop's answer is the LAST attempt's outcome. An attempt that
|
||||
// ends in an indeterminate commit (its COMMIT cut by the context, so the write
|
||||
// may have landed) or in a plain failure is returned unchanged even when the
|
||||
// context ended during it — never replaced by the earlier, rolled-back
|
||||
// conflict, which would tell the caller nothing was written.
|
||||
func TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome(t *testing.T) {
|
||||
for name, last := range map[string]error{
|
||||
"indeterminate commit": fmt.Errorf("sql commit (regular): %w: %w", context.Canceled, ErrCommitIndeterminate),
|
||||
"plain failure": errors.New("validation: dependency would create a cycle"),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
tr := &scriptedTransactor{outcomes: []error{conflict(1), last}}
|
||||
calls := 0
|
||||
err := RunReplayableTransaction(ctx, tr, "bd: last outcome", func(Transaction) error {
|
||||
calls++
|
||||
if calls == 2 {
|
||||
cancel() // the context ends during the replayed attempt
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if !errors.Is(err, last) {
|
||||
t.Fatalf("err = %v, want the last attempt's own outcome %v", err, last)
|
||||
}
|
||||
if errors.Is(last, ErrCommitIndeterminate) && errors.Is(err, ErrTransactionConflict) {
|
||||
t.Fatalf("an indeterminate outcome was reported as a rolled-back conflict: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue