diff --git a/CHANGELOG.md b/CHANGELOG.md index c336272f1..25f72cb7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The #6716 fan-in stall is fixed on the proxied-server route and the + remaining store routes.** Two blockers of one dependent taken away at the + same time (parallel workers closing both, or a close racing a + `bd dep remove` or a delete of the other) could leave the dependent + `is_blocked=1` and missing from `bd ready` until `bd recompute-blocked` + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The post-commit + recheck the Dolt store write transactions gained for that issue now also + runs on every write the proxied-server (uow/domain-db) route serves, which + under `--proxied-server` — the default topology — is `bd close` (single + and batch), `bd update --status`, `bd dep remove`, `bd delete`, `bd batch` + and `bd serve`; on `RunInTransaction` (`bd batch` direct, `bd cook`, + `bd mol squash`/`burn`, and SDK callers that close or update inside a + transaction); and on the wisp close, update and delete writers and the + legacy dependency removal behind `bd duplicates --merge`. Demote-to-wisp + moves a row between planes without taking a blocker away and is unchanged. + - **An explicit `BEADS_DIR` is authoritative during workspace discovery.** When `BEADS_DIR` named a directory without project files yet (missing, empty, or not initialized), discovery ignored it and walked up from the @@ -183,15 +199,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 update to an inactive status, a dependency removal, a delete) and runs through a Dolt store write transaction now rechecks the dependents it recomputed once that transaction has committed - ([#6716](https://github.com/gastownhall/beads/issues/6716)). Writes that - reach the database another way still need the `bd doctor` / - `bd recompute-blocked` repair they needed before: `bd batch` (on both its - plain and its proxied transaction), `bd cook`, `bd mol squash`, - `bd mol burn`, `bd duplicates --merge`, the wisp writes — closes, updates, - deletes and demote-to-wisp — and every write served through the - proxied-server (uow/domain-db) route, which under `--proxied-server` is the - ordinary single verbs as well: `bd close`, `bd update`, `bd delete` and - `bd dep remove`. + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The + proxied-server route and the remaining store routes are covered by the + entry above. - **`bd list --watch --format` is refused instead of silently dropping the diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 96a4ff73e..85e633049 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -533,6 +533,7 @@ go_test( "cli_fast_test.go", "close_embedded_test.go", "close_failure_report_test.go", + "close_fanin_proxied_integration_test.go", "close_gate_test.go", "close_last_touched_test.go", "close_proxied_integration_test.go", diff --git a/cmd/bd/close_fanin_proxied_integration_test.go b/cmd/bd/close_fanin_proxied_integration_test.go new file mode 100644 index 000000000..ad15ff1a7 --- /dev/null +++ b/cmd/bd/close_fanin_proxied_integration_test.go @@ -0,0 +1,84 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedServerCloseSiblingBlockersFanIn is gastownhall/beads#6716 end to +// end on the proxied CLI: the fan-in of a formula whose parallel workers +// close the two blockers of one step at the same time. Each round closes the +// blockers with two concurrent `bd close` processes and then requires the +// dependent in `bd ready`. Without the post-commit recheck on the uow path a +// round whose closes overlap leaves the dependent is_blocked=1 and missing +// from ready work. The interleaving is not pinned here (see +// TestProxiedBlockedRecheckAfterRacingUnblocks in internal/storage/uow for +// the deterministic race); this proves the settled CLI outcome. +func TestProxiedServerCloseSiblingBlockersFanIn(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "fan") + + const rounds = 6 + for round := 0; round < rounds; round++ { + a := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker a", round)).ID + b := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker b", round)).ID + c := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d fan-in", round)).ID + for _, blocker := range []string{a, b} { + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", c, blocker); err != nil { + t.Fatalf("round %d: dep add %s %s: %v\n%s", round, c, blocker, err, out) + } + } + + var wg sync.WaitGroup + errs := make([]string, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + cmd := exec.Command(bd, "close", id) + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + errs[i] = fmt.Sprintf("close %s: %v\n%s", id, err, stderr.String()) + } + }() + } + wg.Wait() + if joined := strings.TrimSpace(strings.Join(errs, "")); joined != "" { + t.Fatalf("round %d: %s", round, joined) + } + + out, err := bdProxiedRun(t, bd, p.dir, "ready", "--json", "--limit", "0") + if err != nil { + t.Fatalf("round %d: bd ready: %v\n%s", round, err, out) + } + start := bytes.IndexByte(out, '[') + if start < 0 { + t.Fatalf("round %d: bd ready printed no JSON array: %s", round, out) + } + var ready []types.Issue + if err := json.Unmarshal(out[start:], &ready); err != nil { + t.Fatalf("round %d: parse bd ready: %v\n%s", round, err, out) + } + found := false + for _, issue := range ready { + found = found || issue.ID == c + } + if !found { + t.Fatalf("round %d: %s has both blockers closed but is missing from bd ready", round, c) + } + } +} diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 06f0b0048..8934919f9 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "blocked_recheck_raw.go", "blockingannotator.go", "bootstrap.go", "bootstrapper.go", @@ -137,7 +138,9 @@ go_test( "blocked_consistency_test.go", "blocked_full_repair_test.go", "blocked_merge_test.go", + "blocked_recheck_phase_test.go", "blocked_recheck_publication_test.go", + "blocked_recheck_raw_test.go", "blocked_recompute_guard_test.go", "blocking_annotator_contract_test.go", "bootstrap_test.go", diff --git a/internal/storage/dolt/blocked_recheck_phase_test.go b/internal/storage/dolt/blocked_recheck_phase_test.go new file mode 100644 index 000000000..337073804 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_phase_test.go @@ -0,0 +1,70 @@ +package dolt + +import ( + "context" + "errors" + "slices" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// scopeAndRecordDependent scopes both of tx's SQL transactions for the +// blocked recheck, as runDoltTransactionRecording does, and records one +// dependent on each, as a close on each plane would. +func scopeAndRecordDependent(t *testing.T, tx *doltTransaction) { + t.Helper() + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.regularTx)) + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.ignoredTx)) + issueops.NoteStatusChangeBlockedRecheck(tx.regularTx, "rp-a", string(types.StatusClosed), []string{"rp-c"}, nil) + issueops.NoteStatusChangeBlockedRecheck(tx.ignoredTx, "rp-wa", string(types.StatusClosed), nil, []string{"rp-wc"}) +} + +// TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails pins the +// RunInTransaction half of #6716 where the regular SQL commit lands and the +// Dolt commit after it fails. The rows are committed, the error is +// ErrCommitIndeterminate and the write is never replayed, so the recorded +// dependents must still reach the post-commit recheck. +func TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + tx.dirty.MarkDirty("issues") + regularMock.ExpectCommit() + regularMock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM dolt_status s"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + regularMock.ExpectExec("CALL DOLT_ADD\\(\\?\\)").WithArgs("issues"). + WillReturnError(errors.New("invalid connection")) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: dolt commit fails") + if !errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want ErrCommitIndeterminate", err) + } + if !slices.Equal(pending.IssueIDs, []string{"rp-c"}) || !slices.Equal(pending.WispIDs, []string{"rp-wc"}) { + t.Fatalf("pending = %+v, want the dependents recorded on both transactions", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} + +// TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused is the +// other side: a regular SQL commit the server refused rolled back, so there +// is nothing committed to recheck. +func TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + regularMock.ExpectCommit().WillReturnError(&mysql.MySQLError{Number: 1213, Message: "serialization failure"}) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: sql commit refused") + if err == nil || errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want a determinate commit failure", err) + } + if !pending.Empty() { + t.Fatalf("pending = %+v, want nothing after a refused SQL commit", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} diff --git a/internal/storage/dolt/blocked_recheck_raw.go b/internal/storage/dolt/blocked_recheck_raw.go new file mode 100644 index 000000000..cd9130ba9 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw.go @@ -0,0 +1,30 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// settleBlockedRecheck runs the post-commit blocked-state recheck of what a +// committed write recorded (gastownhall/beads#6716) and logs a failure +// instead of returning it: the write is durable, and an error would read as +// "the write did not land" to every caller. +func (s *DoltStore) settleBlockedRecheck(ctx context.Context, pending issueops.BlockedRecheck) { + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) +} + +// commitSQLTxAndRecheck is commitSQLTx for a raw write transaction that was +// scoped with issueops.ScopeBlockedRecheckTransaction: once tx has committed +// it settles the dependents tx's unblocking writes recorded. The wisp writers +// and the legacy dependency removal run on raw transactions outside +// withWriteTx, so without this their writes would record into nothing and +// keep the #6716 race. +func (s *DoltStore) commitSQLTxAndRecheck(ctx context.Context, op string, tx *sql.Tx) error { + if err := s.commitSQLTx(ctx, op, tx); err != nil { + return err + } + s.settleBlockedRecheck(ctx, issueops.TakeBlockedRecheck(tx)) + return nil +} diff --git a/internal/storage/dolt/blocked_recheck_raw_test.go b/internal/storage/dolt/blocked_recheck_raw_test.go new file mode 100644 index 000000000..a7609d68f --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw_test.go @@ -0,0 +1,143 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "slices" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// seedBlockedDependent creates blockers a and b (wisps when wisp is set) and +// the permanent dependent c that both block. +func seedBlockedDependent(t *testing.T, ctx context.Context, store *DoltStore, a, b, c string, wisp bool) { + t.Helper() + for _, id := range []string{a, b} { + if wisp { + createWisp(t, ctx, store, id) + } else { + createPerm(t, ctx, store, id) + } + } + createPerm(t, ctx, store, c) + addDependency(t, ctx, store, c, a, types.DepBlocks) + addDependency(t, ctx, store, c, b, types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", c, true) +} + +// closeOnSeparatePool closes id and commits it on a pool of its own, with the +// ordinary in-transaction recompute. The store's own transaction may hold +// every connection its pool has, so the sibling writer cannot borrow one. +func closeOnSeparatePool(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + db, err := sql.Open("mysql", store.connStr) + if err != nil { + t.Fatalf("open sibling pool: %v", err) + } + defer func() { _ = db.Close() }() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin sibling tx: %v", err) + } + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "sibling", ""); err != nil { + _ = tx.Rollback() + t.Fatalf("sibling close of %s: %v", id, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit sibling close of %s: %v", id, err) + } +} + +func assertReady(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + assertIsBlocked(t, ctx, store, "issues", id, false) + ready, err := store.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if !slices.ContainsFunc(ready, func(i *types.Issue) bool { return i.ID == id }) { + t.Fatalf("%s has no open blockers but is missing from ready work", id) + } +} + +// TestBlockedRecheck_RunInTransactionCommittingLast is gastownhall/beads#6716 +// on DoltStore.RunInTransaction, the storage.Transaction runner `bd batch`, +// `bd cook`, `bd mol squash`/`burn` and SDK callers (Gas City's native store +// closes and status updates) write through. The transaction closes one +// blocker while both are open, a sibling close of the other commits, and the +// transaction commits last: without a post-commit recheck the dependent stays +// blocked. +func TestBlockedRecheck_RunInTransactionCommittingLast(t *testing.T) { + cases := []struct { + name string + wisp bool + write func(ctx context.Context, tx storage.Transaction, id string) error + }{ + {"close", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + {"update to closed", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.UpdateIssue(ctx, id, map[string]interface{}{"status": types.StatusClosed}, "tester") + }}, + {"wisp close", true, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedBlockedDependent(t, ctx, store, "rt-a", "rt-b", "rt-c", tc.wisp) + + if err := store.RunInTransaction(ctx, "close rt-b", func(tx storage.Transaction) error { + if err := tc.write(ctx, tx, "rt-b"); err != nil { + return err + } + closeOnSeparatePool(t, ctx, store, "rt-a") + return nil + }); err != nil { + t.Fatalf("RunInTransaction: %v", err) + } + + assertReady(t, ctx, store, "rt-c") + }) + } +} + +// TestBlockedRecheck_ConcurrentWispCloses races the raw-transaction wisp +// close path (closeWisp) against itself. It cannot pin the interleaving the +// way the tests above do, so it only proves the settled outcome: whichever +// close commits last, the dependent ends up ready. +func TestBlockedRecheck_ConcurrentWispCloses(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + for round := 0; round < 5; round++ { + a, b, c := fmt.Sprintf("rw-%da", round), fmt.Sprintf("rw-%db", round), fmt.Sprintf("rw-%dc", round) + seedBlockedDependent(t, ctx, store, a, b, c, true) + var wg sync.WaitGroup + errs := make([]error, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = store.CloseIssue(ctx, id, "done", "tester", "") + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { + t.Fatalf("round %d close: %v", round, err) + } + } + assertReady(t, ctx, store, c) + } +} diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index 8c5123c65..0cc2ef202 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -110,10 +110,12 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de // method's own history is made of. Uniform beats clever. clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent); err != nil { return err } - if err := s.commitSQLTx(ctx, "commit remove wisp dependency", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit remove wisp dependency", tx); err != nil { return err } return nil @@ -133,6 +135,8 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de // unless activation is bound to THIS transaction. clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() eventWritten, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent) if err != nil { @@ -142,6 +146,10 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de if err := s.commitSQLTx(ctx, "sql commit", tx); err != nil { return err } + // The removal is durable; its dependents are settled after this + // removal's own Dolt commit so the recheck's commit follows it. + pending := issueops.TakeBlockedRecheck(tx) + defer s.settleBlockedRecheck(ctx, pending) // GH#2455: Use explicit DOLT_ADD to avoid sweeping up stale config changes. // Stage events only when RemoveDependencyInTx actually recorded a // dependency_removed event (explicit verb + genuine edge removal). A diff --git a/internal/storage/dolt/journal_scope_completeness_test.go b/internal/storage/dolt/journal_scope_completeness_test.go index 491eae888..0e709acc5 100644 --- a/internal/storage/dolt/journal_scope_completeness_test.go +++ b/internal/storage/dolt/journal_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // scopeCalls are the ways a function binds journal activation to the concrete // transaction it just began: the store helper, or the issueops primitive it -// wraps (runDoltTransaction calls the latter directly). +// wraps (runDoltTransactionRecording calls the latter directly). var scopeCalls = map[string]bool{ "scopeEventsJournalTransaction": true, "ScopeEventsJournalTransaction": true, @@ -88,7 +88,7 @@ func TestEveryRawTxJournalScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // journaling mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are checked here in their own right. // // Reachability is deliberately DIRECT rather than a call-graph fixpoint. diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index eab238b0d..581b2a16d 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -896,20 +896,19 @@ var doltTracer = otel.Tracer("github.com/steveyegge/beads/storage/dolt") // Instruments are registered against the global delegating provider at init time, // so they automatically forward to the real provider once telemetry.Init() runs. var doltMetrics struct { - retryCount metric.Int64Counter - lockWaitMs metric.Float64Histogram - circuitTrips metric.Int64Counter - circuitRejected metric.Int64Counter - serializationErrors metric.Int64Counter - writeRetries metric.Int64Counter - postTxCommitDropped metric.Int64Counter - blockedRecheckDropped metric.Int64Counter - connAcquireMs metric.Float64Histogram - poolWaitCount metric.Int64Counter - poolWaitMs metric.Float64Histogram - claimVerifyLost metric.Int64Counter - claimVerifyRecovered metric.Int64Counter - ignoredTxFreshPool metric.Int64Counter + retryCount metric.Int64Counter + lockWaitMs metric.Float64Histogram + circuitTrips metric.Int64Counter + circuitRejected metric.Int64Counter + serializationErrors metric.Int64Counter + writeRetries metric.Int64Counter + postTxCommitDropped metric.Int64Counter + connAcquireMs metric.Float64Histogram + poolWaitCount metric.Int64Counter + poolWaitMs metric.Float64Histogram + claimVerifyLost metric.Int64Counter + claimVerifyRecovered metric.Int64Counter + ignoredTxFreshPool metric.Int64Counter } func init() { @@ -942,10 +941,6 @@ func init() { metric.WithDescription("Post-tx dolt commits abandoned after retries; the data landed but no dolt commit was minted (change rides the next commit on the branch)"), metric.WithUnit("{commit}"), ) - doltMetrics.blockedRecheckDropped, _ = m.Int64Counter("bd.db.blocked_recheck_dropped", - metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), - metric.WithUnit("{recheck}"), - ) doltMetrics.connAcquireMs, _ = m.Float64Histogram("bd.db.conn_acquire_ms", metric.WithDescription("Time to acquire a pooled connection for a Dolt transaction"), metric.WithUnit("ms"), @@ -1366,8 +1361,7 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche if err == nil { return } - doltMetrics.blockedRecheckDropped.Add(ctx, 1) - log.Printf("warning: %s", issueops.BlockedRecheckFailureMessage(pending, err)) + issueops.ReportBlockedRecheckFailure(ctx, pending, err) } // SetEventsJournalEnabled activates the journal for this store instance only. diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index eebc41a00..8bebf0b9f 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -156,6 +156,21 @@ func (t *doltTransaction) dirtyTableNames() []string { } func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { + pending, err := s.runDoltTransactionRecording(ctx, commitMsg, fn) + // Settled here, after the pinned connection and any borrowed ignored-table + // connection have gone back to the pool: the recheck needs one of its own. + // pending is non-empty only when the regular SQL commit landed or may have + // landed, so this also runs when the Dolt commit after it failed + // (ErrCommitIndeterminate): those rows are committed all the same. + s.settleBlockedRecheck(ctx, pending) + return err +} + +// runDoltTransactionRecording is runDoltTransaction's transaction. Once it has +// committed it hands back the dependents the callback's unblocking writes +// recorded on either of its SQL transactions, for the post-commit +// blocked-state recheck (gastownhall/beads#6716). +func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) (issueops.BlockedRecheck, error) { // Pin a single connection for the entire operation: SQL transaction, // config protection, and DOLT_COMMIT must all run on the same Dolt // session. Each pool connection has an independent working set in Dolt @@ -182,18 +197,18 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn } if err != nil { - return fmt.Errorf("failed to acquire connection: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to acquire connection: %w", err) } defer conn.Close() var currentBranch string if err := conn.QueryRowContext(ctx, "SELECT active_branch()").Scan(¤tBranch); err != nil { - return fmt.Errorf("failed to read active branch: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to read active branch: %w", err) } regularTx, err := conn.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("failed to begin regular tx: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to begin regular tx: %w", err) } // The journal counter and rows must commit in the SAME SQL transaction as @@ -220,7 +235,7 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn ignoredCleanup, ignoredTx, err = s.beginIgnoredTxOnBranch(ctx, currentBranch) if err != nil { _ = regularTx.Rollback() - return err + return issueops.BlockedRecheck{}, err } defer ignoredCleanup() } @@ -237,6 +252,15 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn // update IS the creation. clearVersionScope := issueops.ScopeVersionedHistoryTransaction(regularTx, s.versionedHistoryEnabled.Load()) defer clearVersionScope() + // Unblocking writes record their dependents on whichever transaction they + // ran in: the regular one, or the ignored one wisps use when the journal + // does not pin both planes together. + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(regularTx) + defer clearRecheckScope() + if !journalEnabled { + clearIgnoredRecheckScope := issueops.ScopeBlockedRecheckTransaction(ignoredTx) + defer clearIgnoredRecheckScope() + } tx := &doltTransaction{regularTx: regularTx, ignoredTx: ignoredTx, store: s, journalPinned: journalEnabled} @@ -255,10 +279,34 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn if !journalEnabled { _ = ignoredTx.Rollback() } - return err + return issueops.BlockedRecheck{}, err } - return s.finishDoltTransaction(ctx, conn, tx, commitMsg) + return s.finishDoltTransactionRecording(ctx, conn, tx, commitMsg) +} + +// finishDoltTransactionRecording is finishDoltTransaction that also hands back +// the dependents the transaction's unblocking writes recorded, on either of its +// SQL transactions, for the post-commit blocked-state recheck. +// +// They are handed back with the error when the commit is indeterminate +// (ErrCommitIndeterminate): the regular SQL commit may have landed, or did land +// and only the Dolt commit or the ignored-table commit after it failed. The +// rows are then committed without a caller being able to replay the write, so +// the recheck is the only thing that can settle their dependents; being a full +// recompute of committed state, it is harmless if the commit did not land. +// Any other failure means the regular SQL commit was refused and rolled back, +// and nothing is handed back. +func (s *DoltStore) finishDoltTransactionRecording(ctx context.Context, conn *sql.Conn, tx *doltTransaction, commitMsg string) (issueops.BlockedRecheck, error) { + err := s.finishDoltTransaction(ctx, conn, tx, commitMsg) + if err != nil && !errors.Is(err, ErrCommitIndeterminate) { + return issueops.BlockedRecheck{}, err + } + pending := issueops.TakeBlockedRecheck(tx.regularTx) + if !tx.journalPinned { + pending = pending.Merge(issueops.TakeBlockedRecheck(tx.ignoredTx)) + } + return pending, err } // finishDoltTransaction commits the regular SQL transaction, its associated diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 985492533..89b3ab11c 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // versionScopeCalls are the ways a function binds versioned-history activation // to the concrete transaction it just began: the store helper, or the issueops -// primitive it wraps (runDoltTransaction calls the latter directly). +// primitive it wraps (runDoltTransactionRecording calls the latter directly). var versionScopeCalls = map[string]bool{ "scopeVersionedHistoryTransaction": true, "ScopeVersionedHistoryTransaction": true, @@ -98,7 +98,7 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // versioning mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are covered by TestTxMintingWrappersScopeVersionedHistory below. // Reachability is deliberately DIRECT, for the reasons the journal // guard documents at length. @@ -139,14 +139,15 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { // mutator themselves — the callback does, one frame down. // // They are also where the defect this guard exists for actually lived. -// runDoltTransaction is the entry point for bd create, batch create/import, -// bd graph apply, merge_slot and the CLI transact wrappers, so an unscoped -// runDoltTransaction leaves most of the issues plane unversioned while every +// runDoltTransaction (whose transaction is runDoltTransactionRecording) is the +// entry point for bd create, batch create/import, bd graph apply, merge_slot +// and the CLI transact wrappers, so an unscoped runDoltTransactionRecording +// leaves most of the issues plane unversioned while every // raw-tx site in the scan above still looks perfectly scoped. Deleting its // scope call does NOT fail the scan (verified by deleting it) — which is why // this second arm exists rather than being folded into the first. var txMintingWrappers = []string{ - "DoltStore.runDoltTransaction", + "DoltStore.runDoltTransactionRecording", "DoltStore.commitWriteTx", } diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index 36b6bee66..e8452a674 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -194,12 +194,14 @@ func (s *DoltStore) updateWisp(ctx context.Context, id string, updates map[strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // updateWispChecked updates a wisp with the optional atomic preconditions of @@ -221,6 +223,8 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if opts.ExpectedVersion != nil { if err := issueops.CheckVersionInTx(ctx, tx, id, *opts.ExpectedVersion); err != nil { @@ -234,7 +238,7 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // closeWisp closes a wisp in the wisps table. @@ -249,12 +253,14 @@ func (s *DoltStore) closeWisp(ctx context.Context, id string, reason string, act clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { return err } - return s.commitSQLTx(ctx, "commit close wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx) } // closeWispChecked closes a wisp with the is_blocked guard, mirroring closeWisp @@ -282,13 +288,15 @@ func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() res, err := issueops.CloseIssueCheckedInTx(ctx, tx, id, opts.Reason, actor, opts.Session, opts.Force, opts.ExpectedVersion) if err != nil { return storage.CloseIssueResult{}, err } - if err := s.commitSQLTx(ctx, "commit close wisp", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx); err != nil { return storage.CloseIssueResult{}, err } return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil @@ -304,6 +312,8 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, []string{id}) if aerr != nil { @@ -347,8 +357,9 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after wisp delete for %s: %w", id, err) } + issueops.NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) - return s.commitSQLTx(ctx, "commit delete wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit delete wisp", tx) } // deleteWispBatch permanently removes multiple wisps using one transaction per @@ -397,6 +408,8 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, ids) if aerr != nil { @@ -448,8 +461,9 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return 0, fmt.Errorf("recompute is_blocked after batched wisp delete: %w", err) } + issueops.NoteDeleteBlockedRecheck(tx, ids, "", affectedIssues, affectedWisps) - if err := s.commitSQLTx(ctx, "commit batch wisp delete", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit batch wisp delete", tx); err != nil { return 0, err } diff --git a/internal/storage/domain/db/delete_repo_test.go b/internal/storage/domain/db/delete_repo_test.go index d81095731..d6857d38f 100644 --- a/internal/storage/domain/db/delete_repo_test.go +++ b/internal/storage/domain/db/delete_repo_test.go @@ -237,7 +237,7 @@ func (s *testSuite) issueAffectedByDeletionDependers() { } func (s *testSuite) issueRecomputeIsBlockedEmpty() { - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), nil, nil)) + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, nil, nil)) } func (s *testSuite) issueRecomputeIsBlockedFlips() { @@ -247,7 +247,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { newDep("bd-del-rib-depender", "bd-del-rib-blocker", types.DepBlocks), "tester", domain.DepInsertOpts{})) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) var blocked int s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), @@ -257,7 +257,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { _, err := s.Runner().ExecContext(s.Ctx(), "UPDATE issues SET status = 'closed' WHERE id = ?", "bd-del-rib-blocker") s.Require().NoError(err) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), "SELECT is_blocked FROM issues WHERE id = ?", "bd-del-rib-depender").Scan(&blocked)) diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index ced0d22bc..f6a213a65 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -414,6 +414,7 @@ func (r *dependencySQLRepositoryImpl) Delete(ctx context.Context, issueID, depen if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return domain.DepDeleteResult{}, fmt.Errorf("db: DependencySQLRepository.Delete: recompute is_blocked: %w", err) } + issueops.NoteDependencyRemovalBlockedRecheck(r.runner, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on opts.EmitEvent — a structural removal is as real to a diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index c490a29b6..24f6e847d 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -329,6 +329,9 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("db: Update %s: recompute is_blocked: %w", id, err) } + if !newActive { + issueops.NoteStatusChangeBlockedRecheck(r.runner, id, string(newStatus), affectedIssues, affectedWisps) + } } } // Snapshot only after all derived blocked-state maintenance has completed. @@ -1081,8 +1084,12 @@ func (r *issueSQLRepositoryImpl) AffectedByDeletion(ctx context.Context, issueID return issueops.AffectedByDeletionInTx(ctx, r.runner, issueIDs, wispIDs) } -func (r *issueSQLRepositoryImpl) RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error { - return issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs) +func (r *issueSQLRepositoryImpl) RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error { + if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs); err != nil { + return err + } + issueops.NoteDeleteBlockedRecheck(r.runner, deletedIDs, "", issueIDs, wispIDs) + return nil } func (r *issueSQLRepositoryImpl) AsOf(ctx context.Context, id, ref string) (*types.Issue, error) { diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index a8137338a..c73f2c4d2 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -89,7 +89,10 @@ type IssueSQLRepository interface { FindAllDependents(ctx context.Context, ids []string) ([]string, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) AffectedByDeletion(ctx context.Context, issueIDs, wispIDs []string) (affectedIssues, affectedWisps []string, err error) - RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error + // RecomputeIsBlockedAfterDelete recomputes the blocked state of the + // dependents a delete of deletedIDs affected, and records them for the + // post-commit recheck the unit of work runs (gastownhall/beads#6716). + RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error Close(ctx context.Context, id string, params CloseRowParams, actor string, opts IssueTableOpts) (CloseRowResult, error) CloseChecked(ctx context.Context, id string, params CloseRowParams, actor string, force bool) (CloseRowResult, error) Reopen(ctx context.Context, id string, params ReopenRowParams, actor string, opts IssueTableOpts) (ReopenRowResult, error) diff --git a/internal/storage/domain/issue_delete.go b/internal/storage/domain/issue_delete.go index 442c9f181..d7727546a 100644 --- a/internal/storage/domain/issue_delete.go +++ b/internal/storage/domain/issue_delete.go @@ -211,7 +211,7 @@ func (u *issueUseCaseImpl) deleteMany(ctx context.Context, params DeleteIssuesPa result.ReferencesUpdated = refs } - if err := u.issueRepo.RecomputeIsBlocked(ctx, affectedIssues, affectedWisps); err != nil { + if err := u.issueRepo.RecomputeIsBlockedAfterDelete(ctx, allIDs, affectedIssues, affectedWisps); err != nil { return result, fmt.Errorf("delete: recompute is_blocked: %w", err) } diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 12dd98b64..eddfe312a 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "blocked_consistency.go", "blocked_merge.go", "blocked_recheck.go", + "blocked_recheck_report.go", "blocked_state.go", "blocking_annotation.go", "bootstrap.go", @@ -112,6 +113,8 @@ go_library( "@com_github_dolthub_go_mysql_server//sql", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_gowebpki_jcs//:jcs", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel_metric//:metric", ], ) diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go index fe55241af..98dcd1ba6 100644 --- a/internal/storage/issueops/blocked_recheck.go +++ b/internal/storage/issueops/blocked_recheck.go @@ -39,36 +39,26 @@ import ( // // # What is scoped, and what is not // -// Recording only reaches a recheck through a transaction a store scoped (see -// ScopeBlockedRecheckTransaction), and exactly two write runners scope one: -// DoltStore's commitWriteTx and EmbeddedDoltStore's commitConn. A write that -// reaches this package on any other transaction records nothing and behaves -// exactly as it did before the recheck existed — the skew above survives on -// it. Those routes are known and deliberately out of scope here: +// Recording only reaches a recheck through a transaction a write runner +// scoped (see ScopeBlockedRecheckTransaction). The runners that scope one, +// and settle what it recorded once it has committed, are: // -// - the uow/domain-db routing system (internal/storage/uow, with the -// in-transaction recomputes at internal/storage/domain/db/issue.go and -// dependency.go), which serves `bd batch` when proxied and every write a -// uow provider serves. Scoping it needs more than these two lines: -// doltServerTx.Runner() hands out the pinned *sql.Conn, not the *sql.Tx -// this package keys its scope on, so the key has to be threaded through -// the wrapper — the caveat below, already real rather than hypothetical. -// - storage.Transaction (internal/storage/dolt/transaction.go, reached -// through DoltStore.RunInTransaction on a pinned connection), which is -// how `bd batch` closes, status updates and dep removals land when not -// proxied, and how `bd cook`'s proto-subgraph delete, `bd mol squash` and -// `bd mol burn` delete and close. -// - the legacy DoltStore.RemoveDependencyWithOptions (raw BeginTx in -// internal/storage/dolt/dependencies.go), which `bd duplicates --merge` -// uses to remove parent-child edges. -// - the wisp writers (internal/storage/dolt/wisps.go closeWisp/updateWisp/ -// deleteWisp and ephemeral_routing.go's demote-to-wisp), which run raw -// unscoped transactions. The recompute joins depends_on_wisp_id, so a -// permanent issue blocked by two wisps whose removals race keeps the same -// stale flag. +// - DoltStore's commitWriteTx (withWriteTx/withRetryTx), its storage.Transaction +// runner (RunInTransaction, which scopes both its regular and its ignored +// transaction), and the raw transactions of its wisp writers and of the +// legacy RemoveDependencyWithOptions; +// - EmbeddedDoltStore's commitConn, which every embedded write, including +// RunInTransaction, goes through; +// - the uow Tx (internal/storage/uow), which serves every write a uow +// provider serves: the proxied CLI, `bd batch` when proxied, and bd serve. +// It scopes the pinned *sql.Conn its repositories run on, and the +// domain/db repositories record through the Note* helpers below exactly +// where the issueops write paths do. // -// Every one of these leaves the pre-existing, operator-repairable state -// (`bd doctor`, `bd recompute-blocked`); none is made worse by the recheck. +// A write that reaches this package on any other transaction records nothing +// and behaves exactly as it did before the recheck existed; that leaves the +// operator-repairable state (`bd doctor`, `bd recompute-blocked`) the recheck +// was added to prevent. type BlockedRecheck struct { IssueIDs []string WispIDs []string @@ -228,6 +218,43 @@ func TakeBlockedRecheck(tx DBTX) BlockedRecheck { return taken } +// Merge returns the union of r and other: every id either recorded, and the +// sources and counts of both. A runner that scoped two transactions for one +// write (DoltStore's split regular/ignored transactions) settles them with one +// recheck. +func (r BlockedRecheck) Merge(other BlockedRecheck) BlockedRecheck { + merged := BlockedRecheck{SourceCount: r.SourceCount + other.SourceCount} + merged.IssueIDs, _ = appendRecheckIDs(append([]string(nil), r.IssueIDs...), other.IssueIDs, nil) + merged.WispIDs, _ = appendRecheckIDs(append([]string(nil), r.WispIDs...), other.WispIDs, nil) + merged.Sources, _ = appendRecheckIDs(append([]string(nil), r.Sources...), other.Sources, nil) + if len(merged.Sources) > recheckSourceLimit { + merged.Sources = merged.Sources[:recheckSourceLimit] + } + return merged +} + +// NoteStatusChangeBlockedRecheck records, for the post-commit recheck, the +// dependents a move of id to the inactive newStatus (a close included) +// recomputed in tx. The issue itself is excluded: a concurrent writer +// conflicts on its row rather than racing past it. +func NoteStatusChangeBlockedRecheck(tx DBTX, id, newStatus string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) +} + +// NoteDependencyRemovalBlockedRecheck records the dependents a removal of the +// issueID -> dependsOnID edge recomputed in tx. Nothing is excluded: the +// dependent is exactly the row a racing unblocking write can leave stale. +func NoteDependencyRemovalBlockedRecheck(tx DBTX, issueID, dependsOnID string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) +} + +// NoteDeleteBlockedRecheck records the dependents a delete of deletedIDs +// recomputed in tx, leaving out the deleted rows themselves. scope, when set, +// says where the ids came from ("from "). +func NoteDeleteBlockedRecheck(tx DBTX, deletedIDs []string, scope string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, deleteRecheckLabel(deletedIDs, scope), deletedIDs, affectedIssues, affectedWisps) +} + // noteBlockedRecheck records the dependents a write recomputed in tx. source // labels the write for the recheck's commit message. exclude names ids this // call leaves out: for a close or status update, the issue whose row this diff --git a/internal/storage/issueops/blocked_recheck_report.go b/internal/storage/issueops/blocked_recheck_report.go new file mode 100644 index 000000000..eb09b5ba5 --- /dev/null +++ b/internal/storage/issueops/blocked_recheck_report.go @@ -0,0 +1,39 @@ +package issueops + +import ( + "context" + "log" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// blockedRecheckDropped counts post-commit rechecks that failed. It is +// registered on the storage meter every write runner reports through, against +// the global delegating provider, so it forwards once telemetry.Init() runs. +var blockedRecheckDropped metric.Int64Counter + +func init() { + blockedRecheckDropped, _ = otel.Meter("github.com/steveyegge/beads/storage/dolt").Int64Counter("bd.db.blocked_recheck_dropped", + metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), + metric.WithUnit("{recheck}"), + ) +} + +// ReportBlockedRecheckFailure is the one channel a write runner reports a +// failed post-commit recheck through, instead of returning it: the write it +// followed is committed and durable, and every caller of a store write reads +// an error as "the mutation did not land", so surfacing this one would make +// automated callers retry and double-apply. What is left behind is the stale +// is_blocked flag `bd doctor` and `bd recompute-blocked` repair. +// +// Because the failure stops here, the counter and the line are its only +// trace. The counter is what a fleet alerts on; the line names the rows to +// repair. +func ReportBlockedRecheckFailure(ctx context.Context, pending BlockedRecheck, err error) { + if err == nil { + return + } + blockedRecheckDropped.Add(ctx, 1) + log.Printf("warning: %s", BlockedRecheckFailureMessage(pending, err)) +} diff --git a/internal/storage/issueops/bulk_ops.go b/internal/storage/issueops/bulk_ops.go index e96d5b513..636ae1819 100644 --- a/internal/storage/issueops/bulk_ops.go +++ b/internal/storage/issueops/bulk_ops.go @@ -234,7 +234,7 @@ func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo st if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return int(rowsAffected), fmt.Errorf("recompute is_blocked after source-repo delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(issueIDs, "from "+sourceRepo), issueIDs, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, issueIDs, "from "+sourceRepo, affectedIssues, affectedWisps) return int(rowsAffected), nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 5f571b62d..54b5a4bf6 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,7 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } - noteBlockedRecheck(tx, statusChangeRecheckLabel(id, string(types.StatusClosed)), []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, string(types.StatusClosed), affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal — nor the diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index e0406c8f2..f94f48ec2 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -51,7 +51,7 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) e if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) } - noteBlockedRecheck(tx, deleteRecheckLabel([]string{id}, ""), []string{id}, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) return nil } @@ -392,7 +392,7 @@ func DeleteResolvedSetInTx(ctx context.Context, tx *sql.Tx, set DeletionSet, dry if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return nil, fmt.Errorf("recompute is_blocked after batch delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(set.All, ""), set.All, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, set.All, "", affectedIssues, affectedWisps) return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 3467900af..400824cba 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1143,9 +1143,7 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, return false, fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) - // The dependent is exactly the row a racing unblocking write can leave - // stale, so nothing is excluded from the post-commit recheck. - noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) + NoteDependencyRemovalBlockedRecheck(tx, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying // consumer as one from an explicit dep verb. The same holds for the diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 4a73157c2..44add0cea 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -542,7 +542,7 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string updateResult.IssueRowsChanged = !isWisp || recompute.IssueRowsChanged updateResult.WispRowsChanged = isWisp || recompute.WispRowsChanged if !newActive { - noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, newStatus, affectedIssues, affectedWisps) } } } diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 79d948dab..1a98ccda7 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "blocked_recheck.go", "blockingannotator.go", "bootstrapper.go", "commenter.go", @@ -82,6 +83,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "blocked_recheck_race_test.go", "blocking_annotator_contract_test.go", "bootstrapper_contract_test.go", "claimer_contract_test.go", diff --git a/internal/storage/uow/blocked_recheck.go b/internal/storage/uow/blocked_recheck.go new file mode 100644 index 000000000..4fc2ca8eb --- /dev/null +++ b/internal/storage/uow/blocked_recheck.go @@ -0,0 +1,80 @@ +package uow + +import ( + "context" + "fmt" + + "github.com/cenkalti/backoff/v4" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// recheckBlockedAfterCommit recomputes, on the pinned session and a fresh +// snapshot, the blocked state of the dependents a committed unit of work's +// unblocking writes recorded (gastownhall/beads#6716). The in-transaction +// recompute read that unit of work's start snapshot, so two proxied writes +// racing on the blockers of one dependent each saw the other blocker still in +// place and both committed leaving it blocked and hidden from `bd ready`. A +// recompute over the same ids after commit sees both writes. +// +// It runs no SQL when nothing was recorded. It commits the way the unit of +// work did (commitStatement): a Dolt commit named for the writes when it +// changed an issues row and version commits are not deferred, a plain COMMIT +// otherwise. A serialization loss is retried on a new transaction. +// +// The write it follows is durable, so a failure is reported through +// issueops.ReportBlockedRecheckFailure (the counter and warning the store +// runners use), never returned: an error from Commit tells every caller the +// write did not land. What a failure +// leaves is the stale flag `bd doctor` and `bd recompute-blocked` repair. It +// reports whether the session is clean enough to go back to the pool; false +// means a transaction may still be open on it and it must be poisoned. +func (t *doltServerTx) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) bool { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return true + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = txRetryInitialInterval + bo.MaxElapsedTime = DefaultTxRetryMaxElapsed + clean := true + err := backoff.Retry(func() error { + err := t.recheckBlockedOnce(ctx, pending) + if err == nil || isSerializationError(err) { + // A serialization failure means the server already rolled the + // transaction back, so the session is idle for the next attempt. + return err + } + if _, rbErr := t.conn.ExecContext(ctx, "ROLLBACK;"); rbErr != nil { + clean = false + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) + if err != nil { + issueops.ReportBlockedRecheckFailure(ctx, pending, issueops.BlockedRecheckFailed(err)) + } + return clean +} + +// recheckBlockedOnce is one recheck transaction on the pinned session. +func (t *doltServerTx) recheckBlockedOnce(ctx context.Context, pending issueops.BlockedRecheck) error { + if _, err := t.conn.ExecContext(ctx, "START TRANSACTION;"); err != nil { + return fmt.Errorf("begin recheck: %w", err) + } + result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, t.conn, pending.IssueIDs, pending.WispIDs) + if err != nil { + return err + } + message := "" + if result.IssueRowsChanged { + message = pending.CommitMessage() + } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + return err + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + return err +} diff --git a/internal/storage/uow/blocked_recheck_race_test.go b/internal/storage/uow/blocked_recheck_race_test.go new file mode 100644 index 000000000..c03cfb386 --- /dev/null +++ b/internal/storage/uow/blocked_recheck_race_test.go @@ -0,0 +1,174 @@ +package uow + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedBlockedRecheckAfterRacingUnblocks is gastownhall/beads#6716 on +// the proxied (uow) write path, the one every proxied `bd close`, +// `bd update --status closed`, `bd dep remove`, `bd delete`, proxied batch +// close and bd serve reach. +// +// Two blockers A and B of one dependent C are taken away by two units of +// work whose transactions overlap: the first begins and writes while both +// blockers are open, the second then writes and commits, and the first +// commits last. Each in-transaction recompute saw the other blocker still in +// place, so neither wrote C, and without a post-commit recheck C stays +// is_blocked=1 and hidden from ready work on every committed snapshot. +func TestProxiedBlockedRecheckAfterRacingUnblocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + const prefix = "rbr" + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + kit := newUOWRoleFixtureKit(provider, prefix) + + closeIssue := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssue(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + closeChecked := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssueChecked(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer", false) + return err + } + } + updateClosed := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.IssueUseCase().UpdateIssue(ctx, id, map[string]any{"status": string(types.StatusClosed)}, "racer") + } + } + closeWisp := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseWisp(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + + cases := []struct { + name string + // wispBlockers seeds A and B as wisps instead of issues. + wispBlockers bool + // first and second take blocker A and blocker B away; first commits last. + first, second func(a, b, c string) func(context.Context, UnitOfWork) error + }{ + { + name: "close vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeIssue(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + // The batch closer's per-item close (bd close a b, proxied). + name: "checked close vs checked close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeChecked(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeChecked(b) }, + }, + { + // bd update --status closed, proxied. + name: "update to closed vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return updateClosed(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "dependency removal vs close", + first: func(a, _, c string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.DependencyUseCase().RemoveDependency(ctx, c, a, "racer") + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "delete vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: []string{a}, EnforceCascadePolicy: true, Force: true}, "racer") + return err + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "wisp close vs wisp close", + wispBlockers: true, + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeWisp(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeWisp(b) }, + }, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := string(rune('a' + i)) + a, b, c := prefix+"-"+n+"blka", prefix+"-"+n+"blkb", prefix+"-"+n+"dep" + seed := kit.CreateIssue + if tc.wispBlockers { + seed = kit.CreateWisp + } + for _, id := range []string{a, b} { + if err := seed(ctx, &types.Issue{ID: id, Title: id, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if err := kit.CreateIssue(ctx, &types.Issue{ID: c, Title: c, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", c, err) + } + for _, blocker := range []string{a, b} { + if err := kit.AddDependency(ctx, &types.Dependency{IssueID: c, DependsOnID: blocker, Type: types.DepBlocks}, "seeder"); err != nil { + t.Fatalf("add %s -> %s: %v", c, blocker, err) + } + } + requireBlocked(t, ctx, kit, c, true) + + first, err := provider.NewUOW(ctx) + if err != nil { + t.Fatalf("open first unit of work: %v", err) + } + defer first.Close(ctx) + if err := tc.first(a, b, c)(ctx, first); err != nil { + t.Fatalf("first write: %v", err) + } + + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "second racing write", tc.second(a, b, c)(ctx, uw) + }); err != nil { + t.Fatalf("second write: %v", err) + } + // The second write's own recheck ran while the first was still + // uncommitted, so the dependent is legitimately still blocked. + requireBlocked(t, ctx, kit, c, true) + + if err := first.Commit(ctx, "first racing write"); err != nil { + t.Fatalf("commit first write: %v", err) + } + + requireBlocked(t, ctx, kit, c, false) + ready, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (domain.SearchPage, error) { + return uw.IssueUseCase().GetReadyWork(ctx, types.WorkFilter{}) + }) + if err != nil { + t.Fatalf("ready work: %v", err) + } + if !slices.ContainsFunc(ready.Items, func(i *types.Issue) bool { return i.ID == c }) { + t.Fatalf("%s has no open blockers but is missing from ready work", c) + } + }) + } +} + +func requireBlocked(t *testing.T, ctx context.Context, kit roleFixtureKit, id string, want bool) { + t.Helper() + var blocked bool + if err := kit.QueryScalar(ctx, "SELECT is_blocked FROM issues WHERE id = ?", []any{id}, &blocked); err != nil { + t.Fatalf("read is_blocked of %s: %v", id, err) + } + if blocked != want { + t.Fatalf("%s is_blocked = %v, want %v", id, blocked, want) + } +} diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 7acd42628..e80c119d1 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -193,11 +193,13 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { // UPDATE and the SELECT that must observe it are inside one transaction on // one session. Both scopes are released when the connection is // (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its - // transaction. + // transaction. The blocked-recheck scope is bound and released the same + // way; Commit takes what it recorded once the transaction has committed. return &doltServerTx{ conn: conn, clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()), + clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn), }, nil } diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index a97cf8036..4fd396e7a 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -22,6 +22,10 @@ type doltServerTx struct { // issue-version history: same binding at BeginTx, same release at // releaseConn/poisonConn. clearVersionScope func() + // clearRecheckScope is the same binding for the dependents this + // transaction's unblocking writes record for the post-commit + // blocked-state recheck (gastownhall/beads#6716). + clearRecheckScope func() } var _ Tx = (*doltServerTx)(nil) @@ -34,6 +38,51 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { if t.done { return errors.New("uow: commit: already done") } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + if isSerializationError(err) { + // As below: the server already rolled the transaction back and + // the caller retries the whole unit of work, so leave the + // pinned session in place for the retry. + return err + } + // A failed status check leaves the transaction open on the pinned + // session, exactly like a failed DOLT_COMMIT below: roll back + // before release, or poison the connection. + return t.closeOpenTxAfterFailure(ctx, err) + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + if err == nil { + t.done = true + // The write is durable. Settle the dependents its unblocking writes + // recorded on a fresh snapshot before the pinned session goes back + // to the pool (gastownhall/beads#6716). A recheck failure is logged, + // never returned: the caller must not read a committed write as + // failed and replay it. + if t.recheckBlockedAfterCommit(ctx, issueops.TakeBlockedRecheck(t.conn)) { + t.releaseConn() + } else { + t.poisonConn() + } + return nil + } + if isSerializationError(err) { + // Serialization failures guarantee the transaction was already rolled + // back and the caller retries them, so leave the pinned session in place + // for the retry rather than tearing it down here. + return err + } + // A non-serialization DOLT_COMMIT failure leaves the transaction open on the + // pinned session. Roll it back before releasing the connection so the next + // borrower cannot inherit and implicitly commit the orphaned writes. If the + // rollback also fails the session state is unknown, so poison the connection + // and let the pool discard it instead of handing it out again. + return t.closeOpenTxAfterFailure(ctx, err) +} + +// commitStatement picks the statement that ends the open transaction on the +// pinned session for a commit named message. +func (t *doltServerTx) commitStatement(ctx context.Context, message string) (string, []interface{}, error) { // dolt.auto-commit=batch/off (GH#4995): defer the Dolt version commit to a // later explicit commit point (bd dolt commit), the same as the embedded // and direct SQL-server backends already do at their own commit sites @@ -90,41 +139,15 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { // uow layer lets it surface as a lost-update signal // (lostupdate_dolt_test.go) — swallowing it a layer down could mask a // silently lost write. - pending, perr := issueops.HasPendingChanges(ctx, t.conn) - if perr != nil { - if isSerializationError(perr) { - // As below: the server already rolled the transaction back and - // the caller retries the whole unit of work, so leave the - // pinned session in place for the retry. - return perr - } - // A failed status check leaves the transaction open on the pinned - // session, exactly like a failed DOLT_COMMIT below: roll back - // before release, or poison the connection. - return t.closeOpenTxAfterFailure(ctx, perr) + pending, err := issueops.HasPendingChanges(ctx, t.conn) + if err != nil { + return "", nil, err } if !pending { stmt, args = "COMMIT;", nil } } - _, err := t.conn.ExecContext(ctx, stmt, args...) - if err == nil { - t.done = true - t.releaseConn() - return nil - } - if isSerializationError(err) { - // Serialization failures guarantee the transaction was already rolled - // back and the caller retries them, so leave the pinned session in place - // for the retry rather than tearing it down here. - return err - } - // A non-serialization DOLT_COMMIT failure leaves the transaction open on the - // pinned session. Roll it back before releasing the connection so the next - // borrower cannot inherit and implicitly commit the orphaned writes. If the - // rollback also fails the session state is unknown, so poison the connection - // and let the pool discard it instead of handing it out again. - return t.closeOpenTxAfterFailure(ctx, err) + return stmt, args, nil } // closeOpenTxAfterFailure finishes a Commit attempt that failed with the @@ -175,6 +198,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() t.releaseVersionScope() + t.releaseRecheckScope() if t.conn != nil { _ = t.conn.Close() t.conn = nil @@ -200,6 +224,15 @@ func (t *doltServerTx) releaseVersionScope() { } } +// releaseRecheckScope is releaseJournalScope's counterpart for the +// blocked-recheck scope. Idempotent for the same reason. +func (t *doltServerTx) releaseRecheckScope() { + if t.clearRecheckScope != nil { + t.clearRecheckScope() + t.clearRecheckScope = nil + } +} + // poisonConn discards the pinned session instead of returning it to the pool. // A session whose transaction may still be open must never be reused: because // go-sql-driver's ResetSession only performs a liveness check (no @@ -209,6 +242,7 @@ func (t *doltServerTx) releaseVersionScope() { func (t *doltServerTx) poisonConn() { t.releaseJournalScope() t.releaseVersionScope() + t.releaseRecheckScope() if t.conn == nil { return }