diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go index 073834b07..248c8e606 100644 --- a/cmd/bd/reclaim.go +++ b/cmd/bd/reclaim.go @@ -31,15 +31,31 @@ least this long ago are reclaimed, so a worker briefly paused (GC, clock skew) is not robbed of live work. Run it from a supervisor on a timer with a window of roughly 2× the claim TTL. -By default reclaim is global: every stale lease in the database is fair game. -The scope filters below narrow it, using the same label surface claiming is +By default reclaim covers every stale lease THIS replica granted. The scope +filters below narrow it further, using the same label surface claiming is scoped by (--label / --label-any / --exclude-label), plus --assignee and --id. -Scoping matters on a federated deployment: each replica's view of the other -machine's liveness is stale by up to one sync interval, so an unscoped reaper on -one machine can revert a unit that is alive on the machine that granted its -lease. Point each supervisor's reclaim at its own claim partition and that -cannot happen. Filters AND-combine and never widen the set: a reclaimed lease -must still be stale. +Filters AND-combine and never widen the set: a reclaimed lease must still be +stale. + +Replicas and leases (federated deployments) +------------------------------------------- +A lease is only meaningful on the replica that granted it. Every other +replica's view of the holder's liveness is stale by up to one sync interval, +so a reaper elsewhere can revert a unit that is very much alive over there. +reclaim therefore records the granting replica on each lease and SKIPS a lease +another replica granted, naming it on stderr. Reap it where it was granted; use +--any-replica only when that replica is permanently gone (or when this node was +renamed and its own old leases now look foreign). + +Two invariants the guard cannot enforce for you: + + grace window > sync interval, and lease TTL > sync interval. + +A TTL or grace shorter than the cadence at which replicas exchange state is +meaningless across the bridge — the remote view is a full interval old by +construction. Raise the TTL/grace above the sync interval, never the reverse. +This replica's identity is BEADS_NODE_ID (or node_id in config.yaml), falling +back to the hostname; set it explicitly wherever the hostname is not stable. Examples: bd reclaim # default grace window (2× the lease TTL) @@ -48,7 +64,8 @@ Examples: bd reclaim --label lane-a # only this machine's claim partition bd reclaim --label-any lane-a,lane-b --exclude-label pinned bd reclaim --assignee zelda --assignee epona # only these workers' leases - bd reclaim --id wy-abc --id wy-def # exactly these issues`, + bd reclaim --id wy-abc --id wy-def # exactly these issues + bd reclaim --any-replica # also reap leases granted by a departed replica`, Args: cobra.NoArgs, SilenceUsage: true, SilenceErrors: true, @@ -136,6 +153,8 @@ func registerReclaimScopeFlags(fs *pflag.FlagSet) { fs.StringSliceP("label", "l", nil, "Only reclaim issues with ALL these labels (AND). Can combine with --label-any") fs.StringSlice("label-any", nil, "Only reclaim issues with AT LEAST ONE of these labels (OR). Can combine with --label") fs.StringSlice("exclude-label", nil, "Never reclaim issues carrying ANY of these labels") + fs.Bool("any-replica", false, + "Also reclaim leases granted by ANOTHER replica (unsafe unless that replica is gone; see 'Replicas and leases')") } // reclaimFilterFromFlags maps the scope flags onto a types.ReclaimFilter, @@ -184,6 +203,13 @@ func reclaimFilterFromFlags(cmd *cobra.Command) (types.ReclaimFilter, error) { if filter.ExcludeLabels, err = get("exclude-label"); err != nil { return types.ReclaimFilter{}, err } + // --any-replica is an override, not a scope: it WIDENS the set past the + // granting-replica guard, so it deliberately skips the empty-value hard + // error above (a bool flag has no empty-value hazard) and never counts + // toward "scoped" in the reclaim report. + if filter.AnyReplica, err = cmd.Flags().GetBool("any-replica"); err != nil { + return types.ReclaimFilter{}, fmt.Errorf("--any-replica: %w", err) + } return filter, nil } diff --git a/cmd/bd/reclaim_test.go b/cmd/bd/reclaim_test.go index cd997c5fd..9c400d703 100644 --- a/cmd/bd/reclaim_test.go +++ b/cmd/bd/reclaim_test.go @@ -80,3 +80,37 @@ func TestReclaimFilterFromFlagsRejectsEmptyValue(t *testing.T) { }) } } + +// TestReclaimFilterAnyReplica pins the override flag's parse: off by default +// (the granting-replica guard is armed unless an operator asks for it), on when +// given, and never mistaken for a scope. The empty-value hard error that guards +// the string-slice flags does not apply to a bool — `--any-replica` with no +// value is the flag, not a degenerate scope. +func TestReclaimFilterAnyReplica(t *testing.T) { + cmd := newReclaimTestCmd() + if err := cmd.ParseFlags(nil); err != nil { + t.Fatal(err) + } + filter, err := reclaimFilterFromFlags(cmd) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if filter.AnyReplica { + t.Error("AnyReplica defaulted to true; the replica guard must be armed by default") + } + + cmd = newReclaimTestCmd() + if err := cmd.ParseFlags([]string{"--any-replica", "--label", "lane-a"}); err != nil { + t.Fatal(err) + } + filter, err = reclaimFilterFromFlags(cmd) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !filter.AnyReplica { + t.Error("--any-replica did not set AnyReplica") + } + if filter.IsEmpty() { + t.Error("--label lane-a alongside --any-replica reported an empty (global) filter") + } +} diff --git a/cmd/bd/show_format.go b/cmd/bd/show_format.go index 823eeae59..311018901 100644 --- a/cmd/bd/show_format.go +++ b/cmd/bd/show_format.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" ) @@ -130,6 +131,13 @@ func formatIssueMetadata(issue *types.Issue) string { if issue.HeartbeatAt != nil { leaseLine += fmt.Sprintf(" (heartbeat %s)", formatTimeAgo(*issue.HeartbeatAt)) } + // Granting replica: only worth a reader's attention when it is NOT + // this node, since that is the case where the lease is unenforceable + // here and bd reclaim will decline it. Unknown provenance ("") stays + // silent — it is the pre-wy-jpd3.7 shape, not a fact about the lease. + if node := issue.LeaseGrantedNode; node != "" && node != config.NodeID() { + leaseLine += fmt.Sprintf(" — granted by replica %s", node) + } lines = append(lines, ui.RenderMuted(leaseLine)) } diff --git a/docs/multi-agent/federation.md b/docs/multi-agent/federation.md index 107abb369..9552e216f 100644 --- a/docs/multi-agent/federation.md +++ b/docs/multi-agent/federation.md @@ -182,6 +182,45 @@ them. This enables proper attribution and trust chains across organizations. Remote connectivity is validated on first push/pull operation, not when adding the peer. This allows configuring remotes before infrastructure is ready. +### Leases are per-replica + +A claim lease (`bd ready --claim` + `bd heartbeat`, reaped by `bd reclaim`) is +only meaningful on the replica that granted it. The `leases` table is +clone-local and never replicates; what crosses the bridge is the claim's +*visibility* — `status`/`assignee` on the issue row — and that is stale on +every other replica by up to one sync interval. + +Two rules follow, and a federated deployment owes both: + +1. **Grace window > sync interval, and lease TTL > sync interval.** A TTL or + `bd reclaim --older-than` grace shorter than the cadence at which replicas + exchange state is meaningless across the bridge: the remote view is a full + interval old by construction, so a reaper over there would be judging + liveness from data older than the lease itself. `bd reclaim` defaults its + grace to 2× the lease TTL; raise the TTL (or the grace) above your sync + interval, never shrink the interval to fit them. +2. **Reclaim belongs to the granting replica.** Each lease records the replica + that granted it, and `bd reclaim` skips a lease granted elsewhere, naming + it on stderr. Reap dead workers on the machine that hired them. + +Set this replica's identity explicitly wherever the hostname is not stable +(containers, DHCP renames) or two machines could resolve to the same name: + +```bash +export BEADS_NODE_ID=mini # or: node_id: mini in config.yaml +``` + +It falls back to the hostname, and an empty identity degrades to the old +behavior (every lease treated as local) rather than failing closed — an +upgrade can never strand a lease the reaper could previously recover. Leases +granted before this feature landed carry no replica and are likewise treated +as local until their next heartbeat re-stamps them. + +`bd reclaim --any-replica` disarms the guard. It is for a replica that is +permanently gone (or a node that was renamed and now sees its own old leases +as foreign) — not a normal setting, since only the granting machine has a +first-hand view of whether the holder is alive. + ## Planned Features The following operation has infrastructure support but is not yet exposed as diff --git a/internal/config/config.go b/internal/config/config.go index e2d310420..0dbfa4346 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -180,6 +180,8 @@ func Initialize() error { // Additional environment variables (not prefixed with BD_) _ = v.BindEnv("identity", "BEADS_IDENTITY") // BindEnv only fails with zero args, which can't happen here v.SetDefault("identity", "") + _ = v.BindEnv("node_id", "BEADS_NODE_ID", "BD_NODE_ID") // replica identity; see NodeID + v.SetDefault("node_id", "") // Dolt configuration defaults // Controls whether beads should automatically create Dolt commits after write commands. @@ -881,6 +883,37 @@ func GetIdentity(flagValue string) string { return "unknown" } +// NodeID returns the identity of THIS replica: the machine whose beads +// database grants and enforces leases here. It is deliberately coarser than +// the actor identity — every clone and every worker on one machine shares a +// beads store, and a lease is enforceable exactly as far as that store +// reaches (see issueops.UpsertLeaseInTx; the leases table is dolt_ignored and +// does not replicate). +// +// Resolution order: +// +// 1. BEADS_NODE_ID / BD_NODE_ID env var, or node_id in config.yaml. Set this +// when the hostname is not stable (containers, DHCP renames) or when two +// replicas could resolve to the same hostname. +// 2. os.Hostname(). +// 3. "" — provenance unknown. +// +// An empty result is a first-class value, not an error: it means "this +// deployment cannot name its replicas", and every consumer must degrade to +// the pre-replica-aware behavior rather than fail closed. Reclaim in +// particular treats an unknown-provenance lease as local (see +// issueops.ReclaimExpiredLeasesInTx) so an upgrade can never strand a stale +// lease that the reaper used to be able to recover. +func NodeID() string { + if id := strings.TrimSpace(GetString("node_id")); id != "" { + return id + } + if hostname, err := os.Hostname(); err == nil { + return strings.TrimSpace(hostname) + } + return "" +} + // FederationConfig holds the federation (Dolt remote) configuration. type FederationConfig struct { Remote string // dolthub://org/beads, gs://bucket/beads, s3://bucket/beads diff --git a/internal/storage/domain/db/issue_scan_parity_test.go b/internal/storage/domain/db/issue_scan_parity_test.go index 559c2ffbb..672fc71b8 100644 --- a/internal/storage/domain/db/issue_scan_parity_test.go +++ b/internal/storage/domain/db/issue_scan_parity_test.go @@ -42,7 +42,7 @@ func TestScanIssue_StringTimestamps(t *testing.T) { for i := range cols { cols[i] = strings.TrimSpace(cols[i]) } - require.Len(t, cols, 49) + require.Len(t, cols, 50) row := []driver.Value{ "bd-test.1", nil, "title", "desc", "", "", "", // id..notes @@ -55,10 +55,10 @@ func TestScanIssue_StringTimestamps(t *testing.T) { nil, nil, nil, nil, // event_kind..payload nil, nil, // due_at, defer_until nil, nil, nil, // work_type, source_system, metadata - int64(12345), // row_lock - nil, nil, // lease_expires_at, heartbeat_at + int64(12345), // row_lock + nil, nil, nil, // lease_expires_at, heartbeat_at, granted_node } - require.Len(t, row, 49) + require.Len(t, row, 50) mock.ExpectQuery("SELECT").WillReturnRows(sqlmock.NewRows(cols).AddRow(row...)) diff --git a/internal/storage/embeddeddolt/lease_replica_test.go b/internal/storage/embeddeddolt/lease_replica_test.go new file mode 100644 index 000000000..61a5d6086 --- /dev/null +++ b/internal/storage/embeddeddolt/lease_replica_test.go @@ -0,0 +1,195 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestReclaimRefusesForeignReplicaLease is the wy-jpd3.7 acceptance: a lease +// granted by ANOTHER replica is not reclaimable here, however stale it looks, +// because this node's view of that holder's liveness is stale by up to one +// sync interval. --any-replica is the deliberate override. +// +// One process plays two replicas via issueops.WithNodeID; the lease row that a +// federated deployment materializes locally (RestoreLeaseOnImportInTx, on the +// JSONL interchange) is stood in for by claiming under the remote node's +// identity, which produces the same row shape. +func TestReclaimRefusesForeignReplicaLease(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + te := newTestEnv(t, "leasereplica") + ctx := t.Context() + + issue := &types.Issue{ + ID: "leasereplica-1", + Title: "claimed on the other machine", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := te.store.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + // Replica "mini" grants the lease. Staleness comes from a NEGATIVE grace + // window below (cutoff in the future), not from a short TTL plus a sleep: + // a real-clock race between TTL and test latency is the flake this suite + // does not need. + miniCtx := issueops.WithNodeID(ctx, "mini") + if err := te.store.ClaimIssue(miniCtx, "leasereplica-1", "alice"); err != nil { + t.Fatalf("ClaimIssue: %v", err) + } + + var grantedNode string + te.queryScalar(t, ctx, "SELECT granted_node FROM leases WHERE issue_id = ?", + []any{"leasereplica-1"}, &grantedNode) + if grantedNode != "mini" { + t.Fatalf("granted_node = %q after claim, want %q", grantedNode, "mini") + } + + // The laptop's reaper sees a stale lease and must decline it. + laptopCtx := issueops.WithNodeID(ctx, "laptop") + reclaimed, err := te.store.ReclaimExpiredLeases(laptopCtx, -time.Hour, types.ReclaimFilter{}, "reaper") + if err != nil { + t.Fatalf("ReclaimExpiredLeases from laptop: %v", err) + } + if len(reclaimed) != 0 { + t.Fatalf("laptop reclaimed %+v, want nothing — the lease was granted by mini", reclaimed) + } + got, err := te.store.GetIssue(ctx, "leasereplica-1") + if err != nil { + t.Fatalf("GetIssue: %v", err) + } + if got.Status != types.StatusInProgress || got.Assignee != "alice" { + t.Fatalf("issue = (%s, %q) after a declined reclaim, want (in_progress, alice)", got.Status, got.Assignee) + } + // The lease row itself must survive too: the guard is applied on the + // per-row DELETE as well as the snapshot SELECT, so a declined reclaim + // leaves nothing half-torn-down. + var leaseRows int + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", []any{"leasereplica-1"}, &leaseRows) + if leaseRows != 1 { + t.Fatalf("lease rows = %d after a declined reclaim, want 1", leaseRows) + } + if got.LeaseGrantedNode != "mini" { + t.Errorf("hydrated LeaseGrantedNode = %q, want %q", got.LeaseGrantedNode, "mini") + } + + // The granting replica reaps it fine. + reclaimed, err = te.store.ReclaimExpiredLeases(miniCtx, -time.Hour, types.ReclaimFilter{}, "reaper") + if err != nil { + t.Fatalf("ReclaimExpiredLeases from mini: %v", err) + } + if len(reclaimed) != 1 || reclaimed[0].ID != "leasereplica-1" { + t.Fatalf("mini reclaimed = %+v, want [{leasereplica-1 alice}]", reclaimed) + } +} + +// TestReclaimAnyReplicaOverride pins the escape hatch: when the granting +// replica is gone for good, --any-replica reverts its stale leases anyway. +func TestReclaimAnyReplicaOverride(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + te := newTestEnv(t, "leaseoverride") + ctx := t.Context() + + issue := &types.Issue{ + ID: "leaseoverride-1", + Title: "stranded by a departed replica", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := te.store.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + miniCtx := issueops.WithNodeID(ctx, "mini") + if err := te.store.ClaimIssue(miniCtx, "leaseoverride-1", "alice"); err != nil { + t.Fatalf("ClaimIssue: %v", err) + } + + laptopCtx := issueops.WithNodeID(ctx, "laptop") + reclaimed, err := te.store.ReclaimExpiredLeases(laptopCtx, -time.Hour, types.ReclaimFilter{AnyReplica: true}, "reaper") + if err != nil { + t.Fatalf("ReclaimExpiredLeases --any-replica: %v", err) + } + if len(reclaimed) != 1 || reclaimed[0].ID != "leaseoverride-1" || reclaimed[0].PreviousOwner != "alice" { + t.Fatalf("reclaimed = %+v, want [{leaseoverride-1 alice}]", reclaimed) + } + got, err := te.store.GetIssue(ctx, "leaseoverride-1") + if err != nil { + t.Fatalf("GetIssue: %v", err) + } + if got.Status != types.StatusOpen || got.Assignee != "" { + t.Errorf("issue = (%s, %q) after --any-replica reclaim, want (open, \"\")", got.Status, got.Assignee) + } +} + +// TestUnknownProvenanceLeaseStaysReclaimable pins the fail-open half of the +// guard, which is what makes this upgrade safe: a lease row whose granting +// replica is unknown — a row granted before the column existed, or granted by +// a deployment that cannot name its replicas — must remain reclaimable, or the +// upgrade itself would strand every in-flight claim's recovery. A heartbeat +// through this node backfills the identity (and never overwrites a known one). +func TestUnknownProvenanceLeaseStaysReclaimable(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + te := newTestEnv(t, "leaselegacy") + ctx := t.Context() + + for _, id := range []string{"leaselegacy-1", "leaselegacy-2"} { + issue := &types.Issue{ + ID: id, + Title: "legacy lease row", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := te.store.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + + // An anonymous replica: NodeID resolves to "", exactly what a pre-0016 + // lease row carries. + anonCtx := issueops.WithNodeID(ctx, "") + for _, id := range []string{"leaselegacy-1", "leaselegacy-2"} { + if err := te.store.ClaimIssue(anonCtx, id, "alice"); err != nil { + t.Fatalf("ClaimIssue(%s): %v", id, err) + } + } + + // A heartbeat from a named node adopts the orphan lease. + laptopHeartbeat := issueops.WithNodeID(ctx, "laptop") + if err := te.store.HeartbeatIssue(laptopHeartbeat, "leaselegacy-2", "alice"); err != nil { + t.Fatalf("HeartbeatIssue: %v", err) + } + var adopted string + te.queryScalar(t, ctx, "SELECT granted_node FROM leases WHERE issue_id = ?", []any{"leaselegacy-2"}, &adopted) + if adopted != "laptop" { + t.Errorf("granted_node = %q after a heartbeat on an unknown-provenance lease, want %q", adopted, "laptop") + } + + // The still-anonymous lease is reclaimable from a NAMED node: unknown + // provenance is treated as local, never as foreign. + laptopCtx := issueops.WithNodeID(ctx, "laptop") + reclaimed, err := te.store.ReclaimExpiredLeases(laptopCtx, -time.Hour, types.ReclaimFilter{}, "reaper") + if err != nil { + t.Fatalf("ReclaimExpiredLeases: %v", err) + } + ids := map[string]bool{} + for _, r := range reclaimed { + ids[r.ID] = true + } + if !ids["leaselegacy-1"] { + t.Errorf("reclaimed = %+v, want the unknown-provenance lease leaselegacy-1 reverted", reclaimed) + } + if !ids["leaselegacy-2"] { + t.Errorf("reclaimed = %+v, want the laptop-adopted lease leaselegacy-2 reverted", reclaimed) + } +} diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go index 776aaa048..263e1c4a2 100644 --- a/internal/storage/issueops/lease.go +++ b/internal/storage/issueops/lease.go @@ -5,8 +5,11 @@ import ( "crypto/rand" "encoding/binary" "fmt" + "os" "time" + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/types" @@ -104,6 +107,30 @@ func LeaseTTL(ctx context.Context) time.Duration { return leaseTTL(ctx) } +// nodeIDContextKey overrides config.NodeID() for a single call. Used by tests +// that have to be two replicas at once (one process, one database, two +// granting nodes); unset in normal use, where every call resolves the real +// machine identity. +type nodeIDContextKey struct{} + +// WithNodeID returns a context whose lease writes and reclaim guard treat node +// as this replica's identity instead of config.NodeID(). +func WithNodeID(ctx context.Context, node string) context.Context { + return context.WithValue(ctx, nodeIDContextKey{}, node) +} + +// NodeID resolves the granting-replica identity for the current lease +// operation: the context override (WithNodeID) if present, else the +// deployment's node identity (config.NodeID()). "" means the provenance of a +// lease granted here is unknown — see ReclaimExpiredLeasesInTx for how the +// guard degrades. +func NodeID(ctx context.Context) string { + if node, ok := ctx.Value(nodeIDContextKey{}).(string); ok { + return node + } + return config.NodeID() +} + // UpsertLeaseInTx grants or re-grants the lease on an issue to holder: a // future expiry, a now heartbeat. The lease row lives in the ephemeral leases // table (dolt_ignored on the Dolt backend, bd-lrgn1), NOT on the issues row, @@ -112,6 +139,10 @@ func LeaseTTL(ctx context.Context) time.Duration { // granted them; cross-machine claim VISIBILITY rides status/assignee on the // issues row, which still commits. // +// granted_node records WHICH replica that is (config.NodeID(), wy-jpd3.7). +// A re-grant through this node makes this node the granting replica, so the +// value is rewritten unconditionally alongside holder/granted_at. +// // INVARIANT: a leases row exists if and only if its issue is a live claim // (in_progress with the row's holder as assignee) on this node. Every path // that ends or transfers a claim — close, unclaim, reclaim, delete, a generic @@ -120,14 +151,15 @@ func LeaseTTL(ctx context.Context) time.Duration { // are never leased (see testHeartbeatWisp) and never get a row here. func UpsertLeaseInTx(ctx context.Context, tx DBTX, id, holder string, now time.Time, ttl time.Duration) error { _, err := tx.ExecContext(ctx, ` - INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at) - VALUES (?, ?, ?, ?, ?) + INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at, granted_node) + VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE holder = VALUES(holder), granted_at = VALUES(granted_at), lease_expires_at = VALUES(lease_expires_at), - heartbeat_at = VALUES(heartbeat_at) - `, id, holder, now, now.Add(ttl), now) + heartbeat_at = VALUES(heartbeat_at), + granted_node = VALUES(granted_node) + `, id, holder, now, now.Add(ttl), now, NodeID(ctx)) if err != nil { return fmt.Errorf("upsert lease for %s: %w", id, err) } @@ -161,6 +193,13 @@ func DeleteLeaseInTx(ctx context.Context, tx DBTX, id string) error { // any now-orphaned local lease row so the UpsertLeaseInTx invariant // (lease row ⇔ live claim) holds. // +// A restored lease keeps the granting replica the SNAPSHOT names +// (issue.LeaseGrantedNode), never this node — this is the one path that can +// materialize a foreign lease row locally, and mislabelling it as local is +// exactly the cross-replica reclaim hazard (wy-jpd3.7). A snapshot written by +// a pre-wy-jpd3.7 binary carries no node, which restores as "" (unknown) and +// therefore behaves exactly as it did before this change. +// // Wisps are never leased; callers route them away before calling this. func RestoreLeaseOnImportInTx(ctx context.Context, tx DBTX, issue *types.Issue, isNew bool) error { now := time.Now().UTC() @@ -184,15 +223,16 @@ func RestoreLeaseOnImportInTx(ctx context.Context, tx DBTX, issue *types.Issue, // comparison column and ON DUPLICATE KEY UPDATE assignments are // evaluated in order, so it must be reassigned LAST. _, err := tx.ExecContext(ctx, ` - INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at) - VALUES (?, ?, ?, ?, ?) + INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at, granted_node) + VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE holder = IF(leases.lease_expires_at >= ?, leases.holder, VALUES(holder)), granted_at = IF(leases.lease_expires_at >= ?, leases.granted_at, VALUES(granted_at)), heartbeat_at = IF(leases.lease_expires_at >= ?, leases.heartbeat_at, VALUES(heartbeat_at)), + granted_node = IF(leases.lease_expires_at >= ?, leases.granted_node, VALUES(granted_node)), lease_expires_at = IF(leases.lease_expires_at >= ?, leases.lease_expires_at, VALUES(lease_expires_at)) - `, issue.ID, assignee, grantedAt, *issue.LeaseExpiresAt, heartbeatAt, - now, now, now, now) + `, issue.ID, assignee, grantedAt, *issue.LeaseExpiresAt, heartbeatAt, issue.LeaseGrantedNode, + now, now, now, now, now) if err != nil { return fmt.Errorf("restore lease for %s: %w", issue.ID, err) } @@ -233,10 +273,16 @@ func RestoreLeaseOnImportInTx(ctx context.Context, tx DBTX, issue *types.Issue, //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { now := time.Now().UTC() + // granted_node is backfilled, never overwritten: a lease row whose + // provenance is unknown (granted before ignored migration 0016) is being + // kept alive through THIS node's store, which is the node that can enforce + // it. A row that already names a replica keeps it — a heartbeat proves the + // holder is alive, not that the lease moved. result, err := tx.ExecContext(ctx, ` - UPDATE leases SET lease_expires_at = ?, heartbeat_at = ? + UPDATE leases SET lease_expires_at = ?, heartbeat_at = ?, + granted_node = IF(granted_node = '', ?, granted_node) WHERE issue_id = ? AND holder = ? - `, now.Add(leaseTTL(ctx)), now, id, actor) + `, now.Add(leaseTTL(ctx)), now, NodeID(ctx), id, actor) if err != nil { return fmt.Errorf("failed to heartbeat issue: %w", err) } @@ -285,6 +331,30 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { // replicate), so a reclaim can only recover claims granted through this node — // which is the only place the lease was ever enforceable anyway. // +// REPLICA GUARD (wy-jpd3.7). "Through this node" is now enforced, not merely +// implied by the transport. The one path that can materialize a foreign lease +// row locally is RestoreLeaseOnImportInTx (lease fields round-trip the JSONL +// interchange), and a lease imported from another replica carries a liveness +// view that is stale by up to one sync interval — reverting on it can rob a +// worker that is very much alive over there. So a lease whose granted_node +// POSITIVELY names another replica is skipped, and only an explicit +// filter.AnyReplica (bd reclaim --any-replica) reverts it. +// +// The guard is deliberately fail-open on unknown provenance: granted_node ” +// (a lease row predating ignored migration 0016, or a deployment whose +// config.NodeID() is empty) is treated as local, so an upgrade can never +// strand a stale lease the reaper could previously recover. The invariant the +// guard cannot enforce, and that every federated deployment still owes: +// +// grace window > sync interval (and TTL > sync interval) +// +// A TTL or grace shorter than the cadence at which replicas exchange state is +// meaningless across the bridge — the remote view is stale by a full interval +// by construction, so a reaper on the far side decides on liveness data older +// than the lease it is judging. bd reclaim's default grace is 2× the lease +// TTL; raise the TTL (or the grace) above the sync interval, not the other way +// round. +// // Reclaim only ever touches the permanent issues table: wisps are ephemeral and // are never leased work. Returns the issues it reverted (id + the owner it took // the lease from) so the caller can log/emit recovery events. The caller owns @@ -296,6 +366,63 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { // and the ids it re-checks all came from the already-scoped snapshot, so a // label removed mid-reclaim cannot smuggle an out-of-scope issue into the // revert. +// reclaimReplicaSQL returns the granting-replica predicate for the reclaim +// snapshot and the per-row DELETE, plus its args. Both sites must apply the +// SAME predicate: the DELETE re-checks by id, and skipping the guard there +// would let a foreign lease slip through on the re-check path. +// +// Empty when the guard is disarmed — either explicitly (filter.AnyReplica) or +// because this deployment cannot name itself (localNode == ""), where every +// comparison would be against "" and the guard would degenerate to "reclaim +// only unknown-provenance leases", stranding this node's own work. +func reclaimReplicaSQL(filter types.ReclaimFilter, localNode string) (string, []any) { + if filter.AnyReplica || localNode == "" { + return "", nil + } + // granted_node '' is unknown provenance and stays eligible (fail-open); + // only a lease that positively names a different replica is protected. + return "\n\t\t AND (COALESCE(granted_node, '') = '' OR granted_node = ?)", []any{localNode} +} + +// warnReplica writes a replica-guard audit line to STDERR (never stdout — +// bd reclaim --json owns stdout and a stray line there breaks every machine +// consumer), suppressed only by quiet mode. +func warnReplica(format string, args ...any) { + if debug.IsQuiet() { + return + } + fmt.Fprintf(os.Stderr, format, args...) +} + +// reportForeignSkips names the stale leases the replica guard declined. +// Best-effort and never fatal: this is an audit courtesy on the reclaim path, +// not a correctness step, so a failed query is swallowed rather than allowed +// to abort a reclaim that is otherwise fine. +func reportForeignSkips(ctx context.Context, tx DBTX, cutoff time.Time, filter types.ReclaimFilter, localNode string) { + scopeSQL, scopeArgs := sqlbuild.ReclaimScopeSQL(filter, sqlbuild.IssuesFilterTables, "i") + args := append([]any{cutoff, localNode}, scopeArgs...) + rows, err := tx.QueryContext(ctx, ` + SELECT l.issue_id, l.granted_node, COALESCE(i.assignee, '') FROM leases l + JOIN issues i ON i.id = l.issue_id + WHERE i.status = 'in_progress' + AND l.lease_expires_at < ? + AND COALESCE(l.granted_node, '') NOT IN ('', ?) + `+scopeSQL, args...) + if err != nil { + return + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var id, grantedNode, holder string + if err := rows.Scan(&id, &grantedNode, &holder); err != nil { + return + } + warnReplica("reclaim: skipping %s (held by %s) — lease granted by replica %q, not this node (%q). "+ + "Reap it on that replica, or pass --any-replica if that replica is gone.\n", + id, holder, grantedNode, localNode) + } +} + func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, filter types.ReclaimFilter, actor string) ([]types.ReclaimedLease, error) { // Snapshot the stale set first so we can report exactly which issues we // reverted and record per-issue recovery events. The DELETE below repeats @@ -303,23 +430,31 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi // between the SELECT and the DELETE is simply skipped (0 rows) — it never // appears as reclaimed. scopeSQL, scopeArgs := sqlbuild.ReclaimScopeSQL(filter, sqlbuild.IssuesFilterTables, "i") - args := append([]any{cutoff}, scopeArgs...) + localNode := NodeID(ctx) + replicaSQL, replicaArgs := reclaimReplicaSQL(filter, localNode) + args := append([]any{cutoff}, replicaArgs...) + args = append(args, scopeArgs...) rows, err := tx.QueryContext(ctx, ` - SELECT l.issue_id, COALESCE(i.assignee, '') FROM leases l + SELECT l.issue_id, COALESCE(i.assignee, ''), COALESCE(l.granted_node, '') FROM leases l JOIN issues i ON i.id = l.issue_id WHERE i.status = 'in_progress' AND l.lease_expires_at < ? - `+scopeSQL, args...) + `+replicaSQL+scopeSQL, args...) if err != nil { return nil, fmt.Errorf("scan for stale leases: %w", err) } var stale []types.ReclaimedLease for rows.Next() { var r types.ReclaimedLease - if err := rows.Scan(&r.ID, &r.PreviousOwner); err != nil { + var grantedNode string + if err := rows.Scan(&r.ID, &r.PreviousOwner, &grantedNode); err != nil { _ = rows.Close() return nil, fmt.Errorf("scan stale lease row: %w", err) } + if filter.AnyReplica && grantedNode != "" && grantedNode != localNode { + warnReplica("reclaim: --any-replica reverting %s (held by %s) — lease was granted by replica %q, not this node (%q)\n", + r.ID, r.PreviousOwner, grantedNode, localNode) + } stale = append(stale, r) } if err := rows.Err(); err != nil { @@ -329,6 +464,15 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi if err := rows.Close(); err != nil { return nil, fmt.Errorf("close stale lease rows: %w", err) } + // Audit the other side of the guard: a stale lease this reaper declined + // because another replica granted it. Silence here would look identical to + // "nothing to recover" while a genuinely dead remote worker's unit sits + // in_progress forever, so name it (the operator's remedy is to reap on the + // granting replica, or --any-replica once that replica is gone for good). + if replicaSQL != "" { + reportForeignSkips(ctx, tx, cutoff, filter, localNode) + } + if len(stale) == 0 { return nil, nil } @@ -340,9 +484,10 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi // cannot be clobbered: heartbeat and reclaim contend on this same lease // row, so one of a racing pair is forced to retry, and a winning // rescuer's pushed-out expiry makes this DELETE match nothing. + delArgs := append([]any{r.ID, cutoff}, replicaArgs...) res, err := tx.ExecContext(ctx, ` DELETE FROM leases WHERE issue_id = ? AND lease_expires_at < ? - `, r.ID, cutoff) + `+replicaSQL, delArgs...) if err != nil { return nil, fmt.Errorf("reclaim %s: %w", r.ID, err) } diff --git a/internal/storage/issueops/lease_replica_test.go b/internal/storage/issueops/lease_replica_test.go new file mode 100644 index 000000000..c240b7155 --- /dev/null +++ b/internal/storage/issueops/lease_replica_test.go @@ -0,0 +1,100 @@ +package issueops + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestReclaimReplicaSQL pins the granting-replica predicate: when it is armed, +// what it lets through, and the two ways it disarms. The predicate is the +// whole guard — the snapshot SELECT and the per-row DELETE both append it — +// so its shape is worth a direct test rather than only end-to-end coverage. +func TestReclaimReplicaSQL(t *testing.T) { + tests := []struct { + name string + filter types.ReclaimFilter + localNode string + wantSQL bool + wantArgs []any + }{ + { + name: "armed on a named node", + localNode: "laptop", + wantSQL: true, + wantArgs: []any{"laptop"}, + }, + { + name: "disarmed by --any-replica", + filter: types.ReclaimFilter{AnyReplica: true}, + localNode: "laptop", + }, + { + // A deployment that cannot name itself would otherwise compare + // every lease against "" — i.e. reclaim ONLY unknown-provenance + // leases and strand its own. Degrade to the historical behavior. + name: "disarmed when this node has no identity", + localNode: "", + }, + { + name: "--any-replica wins over a named node", + filter: types.ReclaimFilter{AnyReplica: true, Labels: []string{"lane-a"}}, + localNode: "mini", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sql, args := reclaimReplicaSQL(tt.filter, tt.localNode) + if tt.wantSQL { + if !strings.Contains(sql, "granted_node") { + t.Fatalf("sql = %q, want a granted_node predicate", sql) + } + // Unknown provenance must stay eligible: the guard protects + // only leases that POSITIVELY name another replica. + if !strings.Contains(sql, "= ''") { + t.Errorf("sql = %q, want unknown provenance ('') to stay eligible", sql) + } + if strings.Count(sql, "?") != 1 { + t.Errorf("sql = %q, want exactly one placeholder", sql) + } + if len(args) != 1 || args[0] != tt.wantArgs[0] { + t.Errorf("args = %v, want %v", args, tt.wantArgs) + } + return + } + if sql != "" || args != nil { + t.Errorf("guard armed: sql = %q, args = %v; want disarmed", sql, args) + } + }) + } +} + +// TestReclaimFilterIsEmptyIgnoresAnyReplica pins that --any-replica is an +// override rather than a scope: a reaper that passes only --any-replica is +// still reporting a GLOBAL sweep, and a supervisor auditing its reclaim log +// must not read it as "narrowed to a partition". +func TestReclaimFilterIsEmptyIgnoresAnyReplica(t *testing.T) { + if !(types.ReclaimFilter{AnyReplica: true}).IsEmpty() { + t.Error("ReclaimFilter{AnyReplica: true}.IsEmpty() = false, want true (an override is not a scope)") + } + if (types.ReclaimFilter{AnyReplica: true, Labels: []string{"lane-a"}}).IsEmpty() { + t.Error("a filter with labels reported empty") + } +} + +// TestNodeIDContextOverride pins the test seam that lets one process be two +// replicas. Without an override NodeID falls through to config.NodeID(), which +// is the real machine — an explicitly empty override must NOT fall through, +// since "this node has no identity" is a case the guard has to be able to +// exercise. +func TestNodeIDContextOverride(t *testing.T) { + ctx := t.Context() + if got := NodeID(WithNodeID(ctx, "mini")); got != "mini" { + t.Errorf("NodeID with override = %q, want %q", got, "mini") + } + if got := NodeID(WithNodeID(ctx, "")); got != "" { + t.Errorf("NodeID with empty override = %q, want %q (must not fall through to config)", got, "") + } +} diff --git a/internal/storage/issueops/scan.go b/internal/storage/issueops/scan.go index a05f6e796..43c7b032b 100644 --- a/internal/storage/issueops/scan.go +++ b/internal/storage/issueops/scan.go @@ -31,6 +31,7 @@ func ScanIssueFrom(s IssueScanner, extra ...any) (*types.Issue, error) { var createdAtStr, updatedAtStr sql.NullString // scanned as strings, parsed with format fallbacks var startedAt, closedAt, compactedAt, dueAt, deferUntil sql.NullTime var leaseExpiresAt, heartbeatAt sql.NullTime // lease columns (migration 0054); NULL when no active lease + var leaseGrantedNode sql.NullString // granting replica (ignored migration 0016); NULL when no active lease var estimatedMinutes, originalSize, timeoutNs sql.NullInt64 var createdBy sql.NullString var assignee, externalRef, specID, compactedAtCommit, owner sql.NullString @@ -54,7 +55,7 @@ func ScanIssueFrom(s IssueScanner, extra ...any) (*types.Issue, error) { &eventKind, &actor, &target, &payload, &dueAt, &deferUntil, &workType, &sourceSystem, &metadata, &rowLock, - &leaseExpiresAt, &heartbeatAt, + &leaseExpiresAt, &heartbeatAt, &leaseGrantedNode, } dests = append(dests, extra...) if err := s.Scan(dests...); err != nil { @@ -184,6 +185,9 @@ func ScanIssueFrom(s IssueScanner, extra ...any) (*types.Issue, error) { if heartbeatAt.Valid { issue.HeartbeatAt = &heartbeatAt.Time } + // Granting replica (ignored migration 0016); "" when the lease predates + // the column or the deployment cannot name its replicas. + issue.LeaseGrantedNode = leaseGrantedNode.String return &issue, nil } diff --git a/internal/storage/schema/migrations/ignored/0016_add_lease_granted_node.up.sql b/internal/storage/schema/migrations/ignored/0016_add_lease_granted_node.up.sql new file mode 100644 index 000000000..018f15701 --- /dev/null +++ b/internal/storage/schema/migrations/ignored/0016_add_lease_granted_node.up.sql @@ -0,0 +1,34 @@ +-- Ignored migration 0016: record the granting replica on every lease +-- (wy-jpd3.7). +-- +-- A lease is only meaningful on the replica that granted it. Every other +-- replica's view of the holder's liveness is stale by up to one sync +-- interval, so a reaper running there can revert a unit that is very much +-- alive on the machine that granted its lease. granted_node names that +-- machine (config.NodeID()) so reclaim can refuse a positively-foreign lease +-- instead of relying on the operator to partition every reaper by hand. +-- +-- Lives on the IGNORED track because leases does: the table is dolt_ignored +-- (never replicated, clone-local), materialized by ignored/0012 on fresh +-- clones and by synced 0055 on in-place upgrades. Same shape as +-- ignored/0013: guarded so it is a no-op on a workspace that already carries +-- the column and on one with no leases table yet. +-- +-- Empty string is the "provenance unknown" value, and DEFAULT '' is what +-- pre-existing lease rows adopt. Reclaim treats unknown as local (fail-open) +-- so this upgrade can never strand a stale lease the reaper could previously +-- recover; a heartbeat re-stamps an unknown row with the local node. +SET @needs_add = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'leases') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'leases' + AND COLUMN_NAME = 'granted_node') = 0, + 1, 0 +); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE leases ADD COLUMN granted_node VARCHAR(255) NOT NULL DEFAULT ''''', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/sqlbuild/sqlbuild.go b/internal/storage/sqlbuild/sqlbuild.go index 7427fc297..9ec5af6b1 100644 --- a/internal/storage/sqlbuild/sqlbuild.go +++ b/internal/storage/sqlbuild/sqlbuild.go @@ -50,7 +50,7 @@ const IssueBaseColumns = `id, content_hash, title, description, design, acceptan // live in the ephemeral leases table (bd-lrgn1), not on the issues row, so // every query selecting these must also add LeaseJoin to its FROM clause — // a query that forgets the join fails loudly on the leases.* reference. -const LeaseSelectColumns = `leases.lease_expires_at, leases.heartbeat_at` +const LeaseSelectColumns = `leases.lease_expires_at, leases.heartbeat_at, leases.granted_node` // IssueSelectColumns is the canonical column list for full issue hydration. // Every query that reads a complete types.Issue should use this constant diff --git a/internal/types/types.go b/internal/types/types.go index a5d9f153c..af5229d0f 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -56,6 +56,15 @@ type Issue struct { // below (json:"-", never serialized). LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"` // When the current claim's lease expires HeartbeatAt *time.Time `json:"heartbeat_at,omitempty"` // Last heartbeat from the lease owner + // LeaseGrantedNode names the replica that granted the lease + // (config.NodeID() at claim time). A lease is only enforceable there: on + // any other replica the liveness view is stale by up to one sync interval, + // so reclaim refuses a positively-foreign lease unless explicitly + // overridden. Empty means "provenance unknown" (a pre-0016 lease row, or a + // deployment that cannot name its replicas), which is treated as local. + // It rides the JSONL interchange so an imported lease keeps its true + // granting replica. + LeaseGrantedNode string `json:"lease_granted_node,omitempty"` // ===== Concurrency (Go-only; never serialized) ===== // RowVersion is an opaque optimistic-concurrency token for the library's own @@ -1590,10 +1599,21 @@ type ReclaimFilter struct { Labels []string // AND semantics: issue must have ALL these labels LabelsAny []string // OR semantics: issue must have AT LEAST ONE of these labels ExcludeLabels []string // Exclusion: issue must NOT have ANY of these labels + + // AnyReplica is the one field here that WIDENS rather than narrows: it + // disarms the granting-replica guard, letting this reaper revert a lease + // another replica granted (see issueops.ReclaimExpiredLeasesInTx). It is + // an operator escape hatch for a permanently-departed replica or a + // renamed node, never a normal setting — the machine that granted a lease + // is the only one with a first-hand view of whether its holder is alive. + // It does NOT widen past staleness or past the scope fields above. + AnyReplica bool } // IsEmpty reports whether the filter constrains nothing, i.e. reclaim runs in -// its global, unscoped form. +// its global, unscoped form. AnyReplica is deliberately excluded: it is an +// override, not a scope, so a reaper reporting "scoped" still means "narrowed +// to a partition". func (f ReclaimFilter) IsEmpty() bool { return len(f.IDs) == 0 && len(f.Assignees) == 0 && len(f.Labels) == 0 && len(f.LabelsAny) == 0 && len(f.ExcludeLabels) == 0