From 0520516fb00821c72f71c8270991b2b06c200c4e Mon Sep 17 00:00:00 2001 From: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:39:06 +0000 Subject: [PATCH 01/54] fix(defer): resolve bd defer/undefer IDs once (mc-qec7j) On the non-proxied path, bd defer and bd undefer resolved every argument twice: an all-arguments utils.ResolvePartialIDs pre-pass whose result has been discarded since the daemon removal (6a51223b6f), then a per-argument utils.ResolvePartialID inside the write loop. Since #6302 the resolver prints a stderr notice whenever it expands an abbreviation, so `bd defer ` and `bd undefer ` printed that notice twice. Keep the pre-pass and loop over its result. The pre-pass is what makes one unresolvable argument abort the command with exit 1 before any write; deleting it instead would turn `bd defer ` into a partial write that exits 0. The loop's "Error resolving" branch goes with the second resolution: after a successful pre-pass it could fire only if the workspace changed between the two calls. Tests: a partial-ID subtest in TestEmbeddedDefer and TestEmbeddedUndefer counts the notice (twice before the fix, once after) and pins the abort: an unresolvable second argument fails the command and leaves the first issue unchanged. Both subtests also fail against the delete-the-pre-pass alternative. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --- cmd/bd/defer.go | 13 +++++------ cmd/bd/defer_embedded_test.go | 39 +++++++++++++++++++++++++++++++++ cmd/bd/undefer.go | 13 +++++------ cmd/bd/undefer_embedded_test.go | 28 +++++++++++++++++++++++ 4 files changed, 77 insertions(+), 16 deletions(-) diff --git a/cmd/bd/defer.go b/cmd/bd/defer.go index 0a22b18be..762e02ccd 100644 --- a/cmd/bd/defer.go +++ b/cmd/bd/defer.go @@ -82,7 +82,10 @@ Examples: ctx := rootCtx - _, err := utils.ResolvePartialIDs(ctx, store, args) + // Resolve every argument before any write so one bad ID aborts the + // command. The loop reuses these IDs: resolving again would repeat + // the partial-ID notice (mc-qec7j). + resolvedIDs, err := utils.ResolvePartialIDs(ctx, store, args) if err != nil { return HandleError("%v", err) } @@ -93,13 +96,7 @@ Examples: return HandleErrorWithHint("database not initialized", diagHint()) } - for _, id := range args { - fullID, err := utils.ResolvePartialID(ctx, store, id) - if err != nil { - fmt.Fprintf(os.Stderr, "Error resolving %s: %v\n", id, err) - continue - } - + for _, fullID := range resolvedIDs { updates := map[string]interface{}{ "status": string(types.StatusDeferred), } diff --git a/cmd/bd/defer_embedded_test.go b/cmd/bd/defer_embedded_test.go index 95f50afb6..7f5056749 100644 --- a/cmd/bd/defer_embedded_test.go +++ b/cmd/bd/defer_embedded_test.go @@ -40,6 +40,18 @@ func bdUndefer(t *testing.T, bd, dir string, args ...string) string { return stdout.String() } +// bdStderrAllowError runs bd with the given args and returns stderr and the +// exit error without failing the test. BD_NO_PARTIAL_ID_NOTICE is dropped so +// an inherited opt-out cannot hide the partial-ID notice a caller counts. +func bdStderrAllowError(t *testing.T, bd, dir string, args ...string) (string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = envWithout(bdEnv(dir), "BD_NO_PARTIAL_ID_NOTICE") + _, stderr, err := runCommandBuffers(t, cmd) + return stderr.String(), err +} + // getIssueStatus returns the status of an issue via bd show --json. func getIssueStatus(t *testing.T, bd, dir, id string) string { t.Helper() @@ -140,6 +152,33 @@ func TestEmbeddedDefer(t *testing.T) { t.Errorf("expected 'Deferred' on second defer: %s", out) } }) + + // ===== Partial ID (mc-qec7j) ===== + + t.Run("defer_partial_id_notice_once", func(t *testing.T) { + // An explicit ID keeps the abbreviation unambiguous in this workspace. + issue := bdCreate(t, bd, dir, "Defer partial ID", "--type", "task", "--id", "df-qec7jnotice") + + // Every argument is resolved before any write, so one unresolvable + // argument still aborts the whole command. + if stderr, err := bdStderrAllowError(t, bd, dir, "defer", "qec7jnot", "df-qec7jmissing"); err == nil { + t.Errorf("bd defer with an unresolvable ID succeeded; want a non-zero exit\nstderr:\n%s", stderr) + } + if status := getIssueStatus(t, bd, dir, issue.ID); status != "open" { + t.Errorf("aborted bd defer changed %s to %q; want it left open", issue.ID, status) + } + + stderr, err := bdStderrAllowError(t, bd, dir, "defer", "qec7jnot") + if err != nil { + t.Fatalf("bd defer qec7jnot failed: %v\nstderr:\n%s", err, stderr) + } + if n := strings.Count(stderr, `"qec7jnot" is not an exact issue ID`); n != 1 { + t.Errorf("partial-ID notice printed %d times; want exactly once\nstderr:\n%s", n, stderr) + } + if status := getIssueStatus(t, bd, dir, issue.ID); status != "deferred" { + t.Errorf("expected status=deferred, got %q", status) + } + }) } // TestEmbeddedDeferConcurrent exercises defer operations concurrently. diff --git a/cmd/bd/undefer.go b/cmd/bd/undefer.go index b89907be8..b4b1b313b 100644 --- a/cmd/bd/undefer.go +++ b/cmd/bd/undefer.go @@ -45,7 +45,10 @@ Examples: ctx := rootCtx - _, err := utils.ResolvePartialIDs(ctx, store, args) + // Resolve every argument before any write so one bad ID aborts the + // command. The loop reuses these IDs: resolving again would repeat + // the partial-ID notice (mc-qec7j). + resolvedIDs, err := utils.ResolvePartialIDs(ctx, store, args) if err != nil { return HandleError("%v", err) } @@ -56,13 +59,7 @@ Examples: return HandleErrorWithHint("database not initialized", diagHint()) } - for _, id := range args { - fullID, err := utils.ResolvePartialID(ctx, store, id) - if err != nil { - fmt.Fprintf(os.Stderr, "Error resolving %s: %v\n", id, err) - continue - } - + for _, fullID := range resolvedIDs { issue, err := store.GetIssue(ctx, fullID) if err != nil { fmt.Fprintf(os.Stderr, "Error getting %s: %v\n", fullID, err) diff --git a/cmd/bd/undefer_embedded_test.go b/cmd/bd/undefer_embedded_test.go index 088044a95..4d7bcac6d 100644 --- a/cmd/bd/undefer_embedded_test.go +++ b/cmd/bd/undefer_embedded_test.go @@ -142,6 +142,34 @@ func TestEmbeddedUndefer(t *testing.T) { t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) } }) + + // ===== Partial ID (mc-qec7j) ===== + + t.Run("undefer_partial_id_notice_once", func(t *testing.T) { + // An explicit ID keeps the abbreviation unambiguous in this workspace. + issue := bdCreate(t, bd, dir, "Undefer partial ID", "--type", "task", "--id", "ud-qec7jnotice") + bdDefer(t, bd, dir, issue.ID) + + // Every argument is resolved before any write, so one unresolvable + // argument still aborts the whole command. + if stderr, err := bdStderrAllowError(t, bd, dir, "undefer", "qec7jnot", "ud-qec7jmissing"); err == nil { + t.Errorf("bd undefer with an unresolvable ID succeeded; want a non-zero exit\nstderr:\n%s", stderr) + } + if s := getIssueStatus(t, bd, dir, issue.ID); s != "deferred" { + t.Errorf("aborted bd undefer changed %s to %q; want it left deferred", issue.ID, s) + } + + stderr, err := bdStderrAllowError(t, bd, dir, "undefer", "qec7jnot") + if err != nil { + t.Fatalf("bd undefer qec7jnot failed: %v\nstderr:\n%s", err, stderr) + } + if n := strings.Count(stderr, `"qec7jnot" is not an exact issue ID`); n != 1 { + t.Errorf("partial-ID notice printed %d times; want exactly once\nstderr:\n%s", n, stderr) + } + if s := getIssueStatus(t, bd, dir, issue.ID); s != "open" { + t.Errorf("expected status=open after undefer, got %q", s) + } + }) } // TestEmbeddedUndeferConcurrent exercises undefer operations concurrently. From 91a72ef7b53834977ac23a24154b403534eaa877 Mon Sep 17 00:00:00 2001 From: "gascity-eddie[bot]" <338826946+gascity-eddie[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:10:12 +0000 Subject: [PATCH 02/54] fix(mol): route bond operand resolution like show/update/close (closes #4714) (follow-up to #4720) (#7622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(mol): route bond operand resolution like show/update/close (closes #4714) bd mol bond resolved operands with utils.ResolvePartialID against the local store only, so a bead that lived in a routed store failed with "'' not found (not an issue ID or formula name)" — even though bd show resolved the same ID. This broke `gt sling `, which cooks the default formula and bonds it to a rig-routed bead. - Resolve operands through resolveAndGetIssueForMutation, the same routing fallback bd show/update/close use (local, then prefix-routed rig via routes.jsonl, then contributor auto-routing). Write-intent so a prefix-routed target opens writable and the bond commits where the bead lives. - Run the bond mutation against the store that owns the resolved issue instead of the global local store; reject a bond whose two operands resolve to different stores rather than writing to the wrong database. - Route the --dry-run preview read-only so it previews routed operands too. - Add TestResolveMolBondOperandRouting covering local and routed resolution. Same routing-parity fix as #3608/#3609 for bd close. * fix(mol): fail fast on read-only routed stores, share same-rig bond handles Address the two structural issues from the #4720 review beyond the rebase itself (the rebase already preserves main's unconditional formula fallback from #3874 instead of reintroducing the looksLikeFormulaName gate): - Routed operands resolve but couldn't be bonded: contributor auto-routing deliberately opens stores read-only (they hydrate foreign projects that must never be mutated), yet the bond selected that store and dispatched mutations that only failed at the store layer. Operand resolution now mirrors resolveCloseTargets explicitly (local, then prefix-routed write-intent, then the shared auto-routed read-only handle) and records writability per operand; pickBondStore rejects a bond pinned to a read-only store with a clear message before any mutation is attempted. TestMolBondPrefixRoutedWriteThrough now verifies the actual dependency write commits in the routed database, not just resolution, and TestMolBondAutoRoutedReadOnlyFailsFast covers the fail-fast path. - Same routed database compared as two different stores: pickBondStore compared Go handle identity, but each route-open constructs a fresh store in server mode, so two operands in the same routed rig were wrongly rejected as cross-store. Both operands of a bond now share one routed handle (an already-opened routed store is tried before opening another), and sameBondStore falls back to StoreLocator identity (CLIDir pins the concrete database) for separately-opened handles onto the same database. TestSameBondStoreSeparateHandles covers the two-handle case directly. Co-Authored-By: Claude Fable 5 * fix(mol): validate routed bond homes before mutation Discover bond operands read-only, enforce dry-run parity and cross-store policy before writable reopen, unwrap decorated store identities, preserve actionable route failures, and verify the accepted home after reopen. Co-authored-by: Johan Snyman Agent-Signature: codex-gpt-5.6-sol-high on behalf of matt wilkie * fix(mol): reconcile routed bond discovery with #5253 var validation The rebase onto current main carried two silent regressions the Aug 16 review named: the discovery-based resolution (which replaced resolveOrDescribe on the direct-store path) never validated --var values, so `bd mol bond --dry-run --var bad=...` previewed success for a bond execution refuses — exactly the false preview #5253 fixed — and materializeMolBondOperand wrapped every cook error as "'%s' not found as issue or formula", re-introducing the ErrVarValidation double-wrap #5253 removed from resolveOrCookToSubgraph. discoverMolBondOperand now takes the provided vars and runs formula.ValidateProvidedVars on its formula fallback, so --dry-run and execution reject the bond identically, before any writable reopen. materializeMolBondOperand passes formula.ErrVarValidation through unwrapped, matching resolveOrCookToSubgraph and pour/wisp. Agent-Signature: claude-fable-5 on behalf of R0SEWT Co-Authored-By: Claude Fable 5 * fix(mol): review polish — fs.ErrNotExist, hook-isolated test repos, auto-route coverage Three non-blocking notes from the Aug 16 review: - routed.go: errors.Is(err, fs.ErrNotExist) instead of the legacy os.IsNotExist, so the not-found check keeps working if loadPrefixRoutes ever starts wrapping its open error. - mol_bond_routing_test.go: pin core.hooksPath to .git/hooks after each git init (the initGitRepoAt convention), so a developer or CI box with a global hooksPath override never runs foreign hooks inside test repos. - The widened openAutoRoutedStore behavior (maintainer/default auto-routes open writable for any validated write-intent resolver, not just mol bond) now has both the CHANGELOG sentence and a write-through test: TestMaintainerAutoRouteMutationWritesThrough drives resolveAndGetIssueForMutation — the bd update resolver — through a maintainer auto-route, commits a real update there, and re-reads it via a fresh routed read. Agent-Signature: claude-fable-5 on behalf of R0SEWT Co-Authored-By: Claude Fable 5 * fix(mol): carry main's #5761 formula validation into routed bond discovery Rebase adaptation for #4720 onto main b303d494df. The PR moved the direct (non-proxied) `bd mol bond` path off resolveOrDescribe and resolveOrCookToSubgraph onto discoverMolBondOperand and materializeMolBondOperand. Main's #5761 meanwhile fixed the old pair so a --dry-run fails on exactly what the real bond fails on; the new pair did not have that fix: - discoverMolBondOperand loaded the formula with parser.LoadByName, which never runs Formula.Validate, so a --dry-run previewed a successful bond for a formula the cook rejects (e.g. a waits_for gate with no spawner). It also validated --var values against the unmerged formula, so a var whose enum is declared only in a parent went unchecked. Discovery now resolves the formula with parser.Resolve first, the way resolveAndCookFormulaWithVars does, and wraps both failures the way the cook does, so --dry-run and execution report the same message. - materializeMolBondOperand passed only formula.ErrVarValidation through unwrapped. A formula that fails validation was reported as "not found as issue or formula". formula.ErrValidation now passes through too, as in main's resolveOrCookToSubgraph. TestDiscoverMolBondOperandRejectsWhatMaterializeRejects and TestDiscoverMolBondOperandValidatesVarsAgainstTheMergedFormula assert the two phases against each other, like main's TestBondDryRun* pair does for the proxied route. Also: - The PR's routing tests now pass the `exact` argument that main's #5393 added to resolveViaPrefixRoutingWithAccess. They pass false, which keeps the partial-ID resolution the tests were written against. - The PR's three new cgo test files are added to the bd_test srcs. This is `make bazel-sync` output on the merged tree, and it touches nothing else. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 Co-authored-by: Rody Vilchez <102562850+R0SEWT@users.noreply.github.com> * test(mol): rebuild the RepoContext cache in the PR's role-routing tests Rebase adaptation for #4720 onto main b303d494df. Main's #5243 moved auto-routing role detection onto beads.GetRepoContext(), which caches the repo context once per process; before it, role detection ran routing.DetectUserRole(".") on every call. The PR's auto-routing tests chdir into a fresh repo and set beads.role there, so after the rebase they read the context an earlier test built for its own, now-deleted repo. There beads.role is unset, and the remote-URL fallback finds no origin and answers maintainer: - TestMolBondAutoRoutedReadOnlyFailsFast fails whenever a role-routing test ran before it in the same binary (reproducible alone with -count=2): discovery looks for the contributor operand in the local store and reports it not found. - TestDiscoverMolBondOperandRouting/maintainer_routed_issue and TestMaintainerAutoRouteMutationWritesThrough still pass, but only because that fallback is also maintainer; they never read the beads.role they configure. Each test now calls resetRepoContextCachesForTest after its chdir, as #5243 did for main's own role-routing tests (TestResolveCloseTargets, which TestDiscoverMolBondOperandRouting mirrors, and the routing_read tests). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 Co-authored-by: Rody Vilchez <102562850+R0SEWT@users.noreply.github.com> * fix(routing): open maintainer auto-routes writable for exact-match writes #5393 landed after this PR was written. It moved `bd comment` and `bd comments add` off resolveAndGetIssueForMutation onto a new resolveAndGetIssueForMutationExact. This PR widens the former's auto-route tier for write intent: a maintainer/default target opens writable, while a contributor target stays read-only and MutationForbidden. On the PR's original base that widening covered comment writes. After the rebase the exact resolver still opened every auto-route read-only, so comment writes lost it and the CHANGELOG's "any validated write-intent resolver" no longer held. The exact resolver is meant to keep the mutation resolver's routing fallbacks and only tighten matching (#5393: it "keeps the existing cross-rig prefix-routing and contributor auto-routing fallbacks intact"). Its auto tier now takes the same write-intent path. Matching stays exact, and the abbreviation-refusal precedence is unchanged. TestMaintainerAutoRouteExactMutationWritesThrough pins the change: the routed store must not be read-only, and a comment written through it must commit in the planning database. Without this change the read-only check fails. The write itself still succeeds, because a server-mode read-only open does not refuse it. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 Co-authored-by: Rody Vilchez <102562850+R0SEWT@users.noreply.github.com> * fix(routing): harden routed writes and surface routing failures Review fixes for #4720. bd close resolved its batch's shared auto-route through openRoutedReadStore, so the route opened read-only even though every other write-intent resolver opened it writable. The shared handle now opens through openAutoRoutedStore with write intent, so the close commits where the issue lives. A contributor auto-route now opens writable for write-intent callers, as maintainer and default routes do. The refusal the previous revision added (errContributorRouteMutationForbidden, contributorRouteRefusal, RoutedResult.MutationForbidden) is gone. A contributor's route target is their own planning repository, the one bd create writes to: docs/multi-agent/multi-repo-migration.md and examples/contributor-workflow/README.md have the contributor claim and close planning issues from the fork, and GH#2345, beads-0km and GH#3608 made bd show, update and close reach those issues. The read-only rule came from #4389's review fixes (e8ae7a291a), which kept "contributor auto-routing" read-only to preserve the bd-6dnrw.32/GH#3231 guarantee that "routed reads never mutate a foreign project's history". That guarantee is about reads, and the planning repository is not a foreign project. Base therefore split by backend: a server-mode bd update, close or comment of a planning issue committed, because a read-only server store accepts the write, while the embedded store refused it with "store is read-only". Refusing in every resolver would have broken the documented workflow on both backends. bd mol bond was the only command that checked MutationForbidden, so its check goes too, and a bond of planning issues commits in the planning repository like any other routed write. Reads through every route stay read-only. This reverses the previous revision's stance and is flagged here as a maintainer call. bd unclaim now resolves its issue through resolveAndGetIssueForMutation, as bd update --claim does, so a routed issue opens writable; with the read resolver an embedded routed target refused the release with "store is read-only". Like bd close, it commits a routed change itself (commitPendingIfEmbedded), since the post-run auto-commit covers only the local store. bd human respond and dismiss already opened their target writable but never committed it, so on an embedded workspace the close sat uncommitted in the routed repository's working set; they now commit it the same way. resolveAndGetIssueWithRoutingAccess's writablePrefixRoute parameter is renamed writable, since it now governs auto routes too. A routed commit that fails no longer looks like success, and a retry commits it. bd unclaim printed "Unclaimed" (or listed the issue in its --json output) even when the release's routed commit failed; it now reports the commit error and leaves the issue out. The failed commit leaves the release, or the close human respond and dismiss make, in the routed repository's working set. There a retry finds the issue unassigned or already closed and refuses before it reaches its commit, and nothing else commits a routed target. On that refusal unclaim, respond and dismiss now commit the pending write before reporting it, through commitRoutedWrite, which their success paths share. A routed write never creates the target's database. The routed open runs before the lookup it serves, so an embedded open of a repository that had not been bootstrapped yet created an empty database that outlived an id that missed; bd bootstrap then took it for the real one and skipped the clone from sync.remote. openRoutedTargetStore's writable branch now opens through newExistingDoltStoreFromConfig, which refuses an embedded workspace without a database (embeddeddolt.HasRepository) before the sanitize migration can rewrite anything there. A server-mode open already never creates one. resolveAndGetIssueForMutation already returned a routing failure (an unreadable routes.jsonl, a routed store that fails to open) as itself. The exact resolver and bd close still fell through to "no issue found matching", which misstates why the lookup missed. All of them now share isRoutingFailure. The exact resolver checks its abbreviation refusal first, so the auto tier still gets its chance at an exact match. A prefix route whose target has no usable metadata.json now names the route and the file and says why the file could not be used. An auto-routed store now opens the target's own dolt_database, as a prefix route already did. Otherwise BEADS_DOLT_SERVER_DATABASE, which names the launching workspace's database, would point a server-mode open at the wrong database on a shared server. openRoutedTargetStore holds that override for both route kinds. A write refuses a target whose metadata.json cannot be read or parsed or names no dolt_database (autoRouteTargetDatabase) instead of falling back to that variable. A target with no metadata.json at all opens, read or written, the only embedded database it holds (embeddeddolt.Databases), or the default database (configfile.DefaultDoltDatabase) when it holds none or several. Contributor setup leaves the planning repository without one until the first routed bd create writes it, and names its database the way any workspace without metadata.json does, BEADS_DOLT_SERVER_DATABASE first. The open would otherwise resolve that name again from the launcher's variable, so when the two differed a read that missed failed with "database not found" and a write that missed created a database beside the one setup created. A server-mode target holds no embedded database, so with the variable set such a route now opens the default database rather than the launcher's, which is what it opens without the variable. A read of a target whose metadata.json is unreadable or names no database keeps the lenient lookup it had. bd mol bond's routed home now gets the external-dependency policy and nothing else, as wireExternalDependencyPolicy asks of routed stores. The launching workspace's hooks and telemetry no longer wrap writes into another rig. runMolBond's tail is split into planMolBond, openMolBondHome, materializeMolBondOperands and bondMolBondOperands, which takes it from cyclomatic 26 down to 10 or less per function. getIssueWithRouting and the resolveViaAutoRouting wrapper had no callers left and are removed. Tests: - TestMaintainerAutoRouteMutationWritesThrough now checks that the routed store is writable. Without that check it passes even against a read-only open, because a server-mode read-only store accepts the write. - TestAutoRouteCloseTargetWriteIntent covers bd close's write intent for a maintainer route. - TestContributorAutoRouteWritesThrough replaces TestContributorAutoRouteRefusesWriteIntent: an update, a comment and a close through a contributor route each open the target writable and commit, a read through the same route stays read-only, and a missing id stays a plain not-found. - TestEmbeddedContributorRoutedWritesCommitInPlanning runs bd update --claim, unclaim, comment, close, human respond and human dismiss through the bd binary in a contributor checkout. Each advances the planning database's HEAD, and the issues end up closed there and absent from the fork's database. The embedded shard manifest gains the test. - TestMolBondContributorAutoRoutedWriteThrough replaces TestMolBondAutoRoutedReadOnlyFailsFast: bd mol bond's pipeline bonds two planning issues through a contributor route, and a fresh routed read sees the dependency; setupAutoRoute now seeds extra planning issues for it. TestDiscoverMolBondOperandRouting's contributor case now expects the writable reopen. - TestResolveCloseTargets keeps base's contributor cases (beads-0km); the previous revision had moved them to routing.maintainer. - TestRoutedWriteDoesNotCreateTargetDatabase (in-process) and TestEmbeddedRoutedWriteMissDoesNotCreateTargetDatabase (the bd binary) check that a missed write through a prefix route and through a maintainer auto-route leaves an embedded target that has not been bootstrapped without an embeddeddolt directory. The embedded shard manifest gains the test. - TestRoutedMutationResolversReportRoutingFailures runs each write-intent resolver against an auto route whose store fails to open, an auto route whose target's metadata.json names no dolt_database while BEADS_DOLT_SERVER_DATABASE names the launcher's, and a prefix route whose target cannot load. - TestAutoRouteTargetDatabase covers a write and a read of a target whose metadata.json names a database, is missing, names none, or cannot be parsed; a missing one yields the target's only embedded database, or the default database when it holds none or several, either way. TestDatabases covers the listing, which applies HasRepository's .dolt marker probe to each database. - TestAutoRouteMissWithoutMetadataIgnoresEnv runs a missed read and a missed write through an auto-route to a target that has one database but no metadata.json: the default database while BEADS_DOLT_SERVER_DATABASE names the launcher's, and a database named after the launcher's with the variable set and unset at use. Each miss must report not found and leave the target holding only its database. - TestRoutedWriteRetryCommitsFailedCommit fails the routed commit of bd unclaim (text and --json), human respond and human dismiss with an invalid --dolt-auto-commit. The run must fail with nothing on stdout and leave the routed repository's HEAD as it was; a retry must report its refusal and commit the write there. - TestAutoRouteOpensTargetDatabaseDespiteEnv checks the target-database change. - The auto-routing tests clear BEADS_ROUTING_MODE and BD_ROUTING_MODE through neutralizeRoutingModeEnv. Viper ranks them above the routing.mode the tests write, so a host that exports either var never routes. bdEnv, which builds the bd binary's environment in the embedded tests, now drops BD_ROUTING_MODE as it drops BEADS_*; agent hosts export it, and TestEmbeddedContributorRoutedWritesCommitInPlanning failed on such a host. TestEmbeddedRoutedWriteMissDoesNotCreateTargetDatabase no longer needs its own envWithout. - Reverting each fix turns its test red: the contributor read-only clamp (the contributor tests above), unclaim's resolver or commit and the respond and dismiss commits (the matching step of the embedded test), and each autoRouteTargetDatabase rule. So does dropping unclaim's, respond's or dismiss's retry commit, or unclaim's skip of a release whose commit failed (TestRoutedWriteRetryCommitsFailedCommit); either earlier no-metadata.json rule, the variable, which leaves the stray database the review reported, or a fixed default database, which fails the read of a target setup named after the variable (TestAutoRouteMissWithoutMetadataIgnoresEnv and TestAutoRouteTargetDatabase), and taking the first of several databases (TestAutoRouteTargetDatabase); and bdEnv's BD_ROUTING_MODE drop, on a host that exports BD_ROUTING_MODE=off (both embedded tests). The CHANGELOG entry now records that every auto-route opens writable for the commands that change an existing issue, the unclaim and human respond/dismiss commits, that a routed write never creates the target's database, and the target-database and routing-failure changes. docs/multi-agent/routing.md says that write commands follow every route, contributor, maintainer and default alike, never create the routed repository's database, and refuse a target whose metadata.json is unreadable or names no dolt_database. Both say that an auto-route opens a target with no metadata.json at the one embedded database it holds, or its default database when it holds none or several, while a prefix route always needs the target's dolt_database. That page documents behavior that is not in a release yet, which the docs release-pin decision (engdocs/decisions/2026-07-17-docs-release-pin.md §1) would leave undocumented. The review asked for docs that match the behavior, as feature PRs #6508, #5855, #5997 and #5092 also did, so it is flagged here as a maintainer call. Open, and not changed here. bd close has the same retry gap on base: a re-close of an issue that a failed routed commit left closed counts as already closed and never commits that store (mutatedStores). commitPendingIfEmbedded gates on the launcher's mode (isEmbeddedMode), not the routed store's, so a server-mode launcher that opens an embedded routed repository would leave the write uncommitted there. That gate is shared by every routed write command and is not exercised here. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 Co-authored-by: Rody Vilchez <102562850+R0SEWT@users.noreply.github.com> * ci(shards): list TestDatabases in the 40-shard embedded storage manifest The routed-write hardening added TestDatabases to internal/storage/embeddeddolt/detection_test.go without a slot in the Bazel-only 40-shard block of .github/scripts/embedded-storage-test-shards.txt, so gen_embedded_storage_shard_manifest.py 40 --weights=duration --check reported it missing and TestStorageEmbeddedShardManifestGeneratorNotStale (go test ./scripts) failed. Regenerated with the repository command (gen_embedded_storage_shard_manifest.py 40 --weights=duration --write); the incremental pack adds exactly one line, on shard 25. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 Co-authored-by: Rody Vilchez <102562850+R0SEWT@users.noreply.github.com> --------- Co-authored-by: Rody Vilchez <102562850+R0SEWT@users.noreply.github.com> Co-authored-by: Claude Fable 5 Co-authored-by: matt wilkie Co-authored-by: Eddie the Engineer Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- .github/scripts/embedded-cmd-test-shards.txt | 2 + .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 40 + cmd/bd/BUILD.bazel | 3 + cmd/bd/close.go | 132 ++-- cmd/bd/close_routing_test.go | 1 + cmd/bd/contributor_embedded_test.go | 62 ++ cmd/bd/gate.go | 6 +- cmd/bd/human.go | 18 + cmd/bd/init_embedded_test.go | 5 +- cmd/bd/mol_bond.go | 327 ++++++-- cmd/bd/mol_bond_routing_test.go | 514 ++++++++++++ cmd/bd/mol_bond_var_validation_test.go | 198 +++++ cmd/bd/routed.go | 363 +++++---- cmd/bd/routed_mutation_test.go | 730 ++++++++++++++++++ cmd/bd/routed_test.go | 56 +- cmd/bd/routing_read.go | 20 +- cmd/bd/routing_role.go | 3 +- cmd/bd/store_factory.go | 28 +- cmd/bd/store_factory_nocgo.go | 8 + cmd/bd/unclaim.go | 22 +- cmd/bd/update_embedded_test.go | 67 ++ docs/multi-agent/routing.md | 14 +- internal/storage/embeddeddolt/detection.go | 24 +- .../storage/embeddeddolt/detection_test.go | 79 ++ 25 files changed, 2477 insertions(+), 246 deletions(-) create mode 100644 cmd/bd/mol_bond_routing_test.go create mode 100644 cmd/bd/mol_bond_var_validation_test.go create mode 100644 cmd/bd/routed_mutation_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 41e1d8180..3157db001 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -143,7 +143,9 @@ 100 59 TestEmbeddedDeferAutoWake 100 60 TestEmbeddedCommentsCLI +100 60 TestEmbeddedRoutedWriteMissDoesNotCreateTargetDatabase +100 61 TestEmbeddedContributorRoutedWritesCommitInPlanning 100 61 TestEmbeddedUpdateRoutedStoreCommitsTargetHead 100 62 TestEmbeddedGateCheckSkipsTemplateGates diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 4401f3cfe..c8e48d224 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -388,6 +388,7 @@ 40 24 TestReopenIssue 40 25 TestBatchGetterContract +40 25 TestDatabases 40 25 TestDualWriteFixtureKitIsWired 40 25 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed 40 25 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses diff --git a/CHANGELOG.md b/CHANGELOG.md index 182c09f9c..d598c0cda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -557,6 +557,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd mol bond ` ignored prefix routing** — operand resolution now uses + the same routing fallback as `bd show`/`bd update`/`bd close`, so a bead that + lives in a routed store (a prefix-routed rig via `routes.jsonl`, or the + contributor planning store) resolves instead of failing with `'' not found + as issue or formula`. Previously `bd mol bond` resolved operands + against the local store only, so `gt sling ` — which cooks the + default formula and bonds it to a rig-routed bead — died at `bd mol bond` even + though `bd show ` resolved the same ID. Resolution is write-intent: the + routed target opens writable and the bond commits where the bead lives. Two + operands that live in the same routed rig share one store handle and bond + normally; only a bond whose operands resolve to genuinely different databases + is rejected rather than written to the wrong one. As part of this change, an + auto-route — contributor, maintainer or default — now opens writable for the + commands that change an existing issue (`bd update`, `bd close`, + `bd comment`, `bd unclaim`, `bd dep` and the other routed mutation commands, + as well as `bd mol bond`), so the write commits where the issue lives: for a + contributor, in the planning repository, as the contributor workflow + documents. Every auto-route used to open read-only, so on an embedded + workspace these writes were refused. `bd unclaim` now opens a routed issue + writable, as `bd update --claim` does, and commits the release there; on an + embedded workspace `bd human respond` and `bd human dismiss` now record a + routed change as a Dolt commit in the routed repository instead of leaving it + uncommitted in that repository's working set. A routed + write, prefix or auto, never creates the target's database: an embedded + repository that has not been bootstrapped yet fails the write instead of + gaining an empty database that `bd bootstrap` would then take for the real + one. An auto-routed store, read or write, now opens the target's own + configured `dolt_database`, as a prefix route already did, even when + `BEADS_DOLT_SERVER_DATABASE` names the current workspace's database. An + auto-routed write refuses a target whose `metadata.json` is unreadable or + names no `dolt_database`, while a target with no `metadata.json` at all (a + planning repository nothing has been created in yet) opens the one embedded + database it holds, or its default database when it holds none or several; a + prefix route still requires the target's `dolt_database`. A + routing failure, such as an unreadable `routes.jsonl` or a + routed store that fails to open, is now reported as itself instead of as + `no issue found matching`. Same routing-parity fix as + [#3608](https://github.com/gastownhall/beads/issues/3608) for `bd close`. + Fixes [#4714](https://github.com/gastownhall/beads/issues/4714). + - **`bd gate check` works over a connected HTTP workspace.** Its open-gate listing built the same raw `SearchIssues` filter `bd gate list` did before S20, so the HTTP backend refused the command before reading a gate and gc's diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 0cfa31cb3..164b7c316 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -995,7 +995,9 @@ go_test( "mol_bond_gwn_test.go", "mol_bond_proxied_integration_test.go", "mol_bond_ref_attach_test.go", + "mol_bond_routing_test.go", "mol_bond_test.go", + "mol_bond_var_validation_test.go", "mol_burn_proxied_integration_test.go", "mol_current_proxied_integration_test.go", "mol_distill_proxied_integration_test.go", @@ -1115,6 +1117,7 @@ go_test( "restore_test.go", "root_context_lifecycle_test.go", "routed_local_near_miss_test.go", + "routed_mutation_test.go", "routed_test.go", "routing_notice_test.go", "routing_read_remote_test.go", diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 0d8760c0c..d642af44b 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -872,10 +872,12 @@ func resolveReasonFile(cmd *cobra.Command, hasExistingReason bool) (string, bool // resolveCloseTargets resolves a batch of partial issue IDs for `bd close`, // preserving input order. For each ID it tries the local store first, then -// explicit prefix routing via routes.jsonl, then a shared contributor-routed -// store. This matches resolveAndGetIssueWithRouting's routing precedence. +// explicit prefix routing via routes.jsonl, then a shared auto-routed store. +// This matches resolveAndGetIssueForMutation's routing precedence and write +// intent, and like it returns a routing failure (an unreadable routes.jsonl, a +// routed store that fails to open) as itself rather than as a not-found. // -// The contributor-routed handle is shared across the batch so bulk close does +// The auto-routed handle is shared across the batch so bulk close does // not repeatedly open the same planning store and every result has a clear store // owner for subsequent close-time checks and writes. // @@ -884,60 +886,92 @@ func resolveReasonFile(cmd *cobra.Command, hasExistingReason bool) (string, bool // no-op for routed-via-shared-handle entries because they don't own the handle. func resolveCloseTargets(ctx context.Context, localStore storage.DoltStorage, ids []string) ([]*RoutedResult, func(), error) { results := make([]*RoutedResult, 0, len(ids)) - var sharedRouted storage.DoltStorage - var sharedRoutedTried bool + shared := &closeAutoRoute{localStore: localStore} cleanup := func() { for _, r := range results { r.Close() } - if sharedRouted != nil { - _ = sharedRouted.Close() - } - } - ensureShared := func() (storage.DoltStorage, error) { - if sharedRouted != nil { - return sharedRouted, nil - } - if sharedRoutedTried { - return nil, fmt.Errorf("no auto-routed store available") - } - sharedRoutedTried = true - rs, routed, _, err := openRoutedReadStore(ctx, localStore) - if err != nil { - return nil, err - } - if !routed { - return nil, fmt.Errorf("no auto-routed store available") - } - sharedRouted = rs - return rs, nil + shared.Close() } for _, id := range ids { - // Local first. - if r, err := resolveAndGetFromStore(ctx, localStore, id, false); err == nil { - results = append(results, r) - continue - } else if !isNotFoundErr(err) { + r, err := resolveCloseTarget(ctx, localStore, shared, id) + if err != nil { cleanup() - return nil, func() {}, fmt.Errorf("resolving ID %s: %w", id, err) + return nil, func() {}, err } - // Write-intent: a prefix-routed target opens writable so the close - // commits on the target head (#4141). Contributor auto-routing below - // stays read-only: it hydrates foreign projects that must not be mutated. - if r, err := resolveViaPrefixRoutingWithAccess(ctx, id, true, false); err == nil { - results = append(results, r) - continue - } - // Contributor auto-routing uses one shared store for the whole batch. - if rs, rerr := ensureShared(); rerr == nil { - if r, err := resolveAndGetFromStore(ctx, rs, id, true); err == nil { - // Per-id RoutedResult does not own the shared handle; cleanup() does. - results = append(results, r) - continue - } - } - cleanup() - return nil, func() {}, fmt.Errorf("resolving ID %s: no issue found matching %q", id, id) + results = append(results, r) } return results, cleanup, nil } + +// resolveCloseTarget resolves one ID of a bd close batch: the local store, +// then a prefix route, then the batch's shared auto-routed store. +func resolveCloseTarget(ctx context.Context, localStore storage.DoltStorage, shared *closeAutoRoute, id string) (*RoutedResult, error) { + // Local first. + r, err := resolveAndGetFromStore(ctx, localStore, id, false) + if err == nil { + return r, nil + } + if !isNotFoundErr(err) { + return nil, fmt.Errorf("resolving ID %s: %w", id, err) + } + // Write-intent: a prefix-routed target opens writable so the close + // commits on the target head (#4141). + r, err = resolveViaPrefixRoutingWithAccess(ctx, id, true, false) + if err == nil { + return r, nil + } + if isRoutingFailure(err) { + return nil, fmt.Errorf("resolving ID %s: %w", id, err) + } + r, err = shared.resolve(ctx, id) + if err == nil { + return r, nil + } + if isRoutingFailure(err) { + return nil, fmt.Errorf("resolving ID %s: %w", id, err) + } + return nil, fmt.Errorf("resolving ID %s: no issue found matching %q", id, id) +} + +// closeAutoRoute is the auto-routed store a bd close batch shares. It opens on +// first use and is closed once, by the batch's cleanup. +type closeAutoRoute struct { + localStore storage.DoltStorage + store storage.DoltStorage + err error + tried bool +} + +// resolve looks id up in the shared auto-routed store, opening it on the first +// call with write intent, as resolveAndGetIssueForMutation does: the target +// opens writable, whichever rule selected it, so the close commits where the +// issue lives. The outcome of that open, failure included, holds for the rest +// of the batch. +func (a *closeAutoRoute) resolve(ctx context.Context, id string) (*RoutedResult, error) { + if !a.tried { + a.tried = true + rs, routed, _, err := openAutoRoutedStore(ctx, a.localStore, true) + switch { + case err != nil: + a.err = err + case !routed: + a.err = fmt.Errorf("%w: no auto-routed store available", errRoutingUnavailable) + default: + a.store = rs + } + } + if a.err != nil { + return nil, a.err + } + // Per-id RoutedResult does not own the shared handle; Close does. + return resolveAndGetFromStore(ctx, a.store, id, true) +} + +// Close closes the shared auto-routed store, if one was opened. +func (a *closeAutoRoute) Close() { + if a.store != nil { + _ = a.store.Close() + a.store = nil + } +} diff --git a/cmd/bd/close_routing_test.go b/cmd/bd/close_routing_test.go index 8073073e9..73efdf514 100644 --- a/cmd/bd/close_routing_test.go +++ b/cmd/bd/close_routing_test.go @@ -69,6 +69,7 @@ func TestResolveCloseTargets(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { initConfigForTest(t) + neutralizeRoutingModeEnv(t) tmpDir := t.TempDir() repoDir := filepath.Join(tmpDir, "repo") diff --git a/cmd/bd/contributor_embedded_test.go b/cmd/bd/contributor_embedded_test.go index 8d58b24e1..eb5964e69 100644 --- a/cmd/bd/contributor_embedded_test.go +++ b/cmd/bd/contributor_embedded_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" ) // The TestEmbeddedContributorCreate* tests exercise the full contributor mode flow in @@ -98,6 +99,67 @@ func TestEmbeddedContributorCreateMultiple(t *testing.T) { }) } +// TestEmbeddedContributorRoutedWritesCommitInPlanning runs, through the bd +// binary in a contributor checkout, the commands that change an issue living in +// the planning repository: each writes the change there and advances the +// planning database's HEAD, as the TestEmbeddedRoutedSiblingWritesCommitTargetHead* +// tests pin for a prefix route. While the contributor auto-route opened its +// target read-only, the embedded store refused every one of these writes. +func TestEmbeddedContributorRoutedWritesCommitInPlanning(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, planningDir := initContributor(t, bd, "cw") + planningBeadsDir := filepath.Join(planningDir, ".beads") + issue := bdCreate(t, bd, dir, "Planning issue") + question := bdCreate(t, bd, dir, "Planning question") + dismissed := bdCreate(t, bd, dir, "Planning request") + + for _, step := range []struct { + name string + args []string + }{ + {"update --claim", []string{"update", issue.ID, "--claim"}}, + {"unclaim", []string{"unclaim", issue.ID}}, + {"comment", []string{"comment", issue.ID, "planning comment"}}, + {"close", []string{"close", issue.ID, "--reason", "done"}}, + {"human respond", []string{"human", "respond", question.ID, "--response", "answered"}}, + {"human dismiss", []string{"human", "dismiss", dismissed.ID, "--reason", "not needed"}}, + } { + before := embeddedCurrentCommit(t, planningBeadsDir, "cw") + bdCommand(t, bd, dir, step.args...) + assertEmbeddedHeadAdvanced(t, planningBeadsDir, "cw", before, step.name) + } + + planningStore := openStore(t, planningBeadsDir, "cw") + for _, id := range []string{issue.ID, question.ID, dismissed.ID} { + got, err := planningStore.GetIssue(t.Context(), id) + if err != nil { + t.Fatalf("GetIssue(%s) in the planning store: %v", id, err) + } + if got.Status != types.StatusClosed { + t.Errorf("%s status in the planning store = %q, want %q", id, got.Status, types.StatusClosed) + } + } + comments, err := planningStore.GetIssueComments(t.Context(), issue.ID) + if err != nil { + t.Fatalf("GetIssueComments: %v", err) + } + found := false + for _, c := range comments { + if c.Text == "planning comment" { + found = true + } + } + if !found { + t.Errorf("the comment did not land in the planning store: %+v", comments) + } + assertIssueNotInStore(t, filepath.Join(dir, ".beads"), "cw", issue.ID) +} + // initContributor sets up a project with contributor mode enabled and returns // the project dir and planning dir paths. The contributor wizard runs via // subprocess with stdin piped to answer the interactive prompts. diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 59f2185bc..47c069ed9 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -1388,9 +1388,9 @@ func beadGateNotFound(err error) bool { // routedBeadGateGetter gives direct-mode gate checks the same local -> prefix // route -> contributor fallback used by other read commands. Routed stores are -// opened read-only and closed before the result is returned. Unlike -// getIssueWithRouting, a route that fails or misses is reported as -// errBeadGateTargetUnconfirmed instead of as the local not-found. +// opened read-only and closed before the result is returned. A route that +// fails or misses is reported as errBeadGateTargetUnconfirmed instead of as +// the local not-found. type routedBeadGateGetter struct { localStore storage.DoltStorage } diff --git a/cmd/bd/human.go b/cmd/bd/human.go index 19fb98907..20c4e50c6 100644 --- a/cmd/bd/human.go +++ b/cmd/bd/human.go @@ -302,6 +302,12 @@ Examples: targetStore := result.Store if issue.Status == "closed" { + // A response whose routed commit failed left its close in the + // target's working set, where this run finds the issue closed. + // Nothing else commits a routed target, so commit the close here. + if err := commitRoutedWrite(ctx, result, "human respond"); err != nil { + return HandleErrorRespectJSON("issue %s is already closed; committing its routed close failed: %v", resolvedID, err) + } return HandleErrorRespectJSON("issue %s is already closed", resolvedID) } @@ -316,6 +322,10 @@ Examples: return HandleErrorRespectJSON("closing bead: %v", err) } + if err := commitRoutedWrite(ctx, result, "human respond"); err != nil { + return HandleErrorRespectJSON("committing response to %s: %v", resolvedID, err) + } + fmt.Printf("%s Bead %s closed with response.\n", ui.RenderPass("✔"), resolvedID) return nil }, @@ -398,6 +408,10 @@ Examples: targetStore := result.Store if issue.Status == "closed" { + // As in respond: commit a dismissal whose routed commit failed. + if err := commitRoutedWrite(ctx, result, "human dismiss"); err != nil { + return HandleErrorRespectJSON("issue %s is already closed; committing its routed close failed: %v", resolvedID, err) + } return HandleErrorRespectJSON("issue %s is already closed", resolvedID) } @@ -407,6 +421,10 @@ Examples: return HandleErrorRespectJSON("closing bead: %v", err) } + if err := commitRoutedWrite(ctx, result, "human dismiss"); err != nil { + return HandleErrorRespectJSON("committing dismissal of %s: %v", resolvedID, err) + } + fmt.Printf("%s Bead %s dismissed.\n", ui.RenderPass("✔"), resolvedID) return nil }, diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index d5c4449b2..dea6ce141 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -110,7 +110,10 @@ func buildEmbeddedBD(t *testing.T) string { func bdEnv(dir string) []string { var env []string for _, e := range os.Environ() { - if strings.HasPrefix(e, "BEADS_") { + // Viper reads BD_ above config.yaml, and agent hosts export + // BD_ROUTING_MODE, which overrides the routing.mode a test's + // workspace holds. + if strings.HasPrefix(e, "BEADS_") || strings.HasPrefix(e, "BD_ROUTING_MODE=") { continue } env = append(env, e) diff --git a/cmd/bd/mol_bond.go b/cmd/bd/mol_bond.go index 4f8a5f2c2..e990bde0d 100644 --- a/cmd/bd/mol_bond.go +++ b/cmd/bd/mol_bond.go @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/internal/formula" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" @@ -118,57 +119,143 @@ func runMolBond(cmd *cobra.Command, args []string) error { return HandleErrorRespectJSON("no database connection") } + plan, err := planMolBond(ctx, store, in) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + defer plan.Close() + + // Dry-run uses the exact same read-only discovery and store-policy + // validation as execution. It stops before writable reopen or formula cook. if in.dryRun { - issueA, formulaA, err := resolveOrDescribe(ctx, store, in.argA, in.vars) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - issueB, formulaB, err := resolveOrDescribe(ctx, store, in.argB, in.vars) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - renderMolBondDryRun(in, issueA, formulaA, issueB, formulaB) + renderMolBondDryRun(in, plan.a.issue, plan.a.formula, plan.b.issue, plan.b.formula) return nil } - subgraphA, cookedA, err := resolveOrCookToSubgraph(ctx, store, in.argA, in.vars) + // Close foreign read handles before opening the one accepted target home + // writable. Rejected cross-store bonds therefore never writable-open a + // foreign rig (and cannot trigger its open-time migrations). + plan.Close() + + activeStore, closeActiveStore, err := openMolBondHome(ctx, store, plan) if err != nil { return HandleErrorRespectJSON("%v", err) } - subgraphB, cookedB, err := resolveOrCookToSubgraph(ctx, store, in.argB, in.vars) + defer closeActiveStore() + + opA, opB, err := materializeMolBondOperands(ctx, activeStore, plan, in.vars) if err != nil { return HandleErrorRespectJSON("%v", err) } - issueA := subgraphA.Root - issueB := subgraphB.Root - aIsProto := issueA.IsTemplate || cookedA - bIsProto := issueB.IsTemplate || cookedB - - var result *BondResult - switch { - case aIsProto && bIsProto: - result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, currentActor()) - case aIsProto && !bIsProto: - if cookedA { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) - } else { - result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) - } - case !aIsProto && bIsProto: - if cookedB { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) - } else { - result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) - } - default: - result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, currentActor()) - } + result, err := bondMolBondOperands(ctx, activeStore, opA, opB, in) if err != nil { return HandleErrorRespectJSON("bonding: %v", err) } - return renderMolBondResult(result, issueA.ID, issueB.ID, in.ephemeral, in.pour) + return renderMolBondResult(result, opA.subgraph.Root.ID, opB.subgraph.Root.ID, in.ephemeral, in.pour) +} + +// molBondPlan is a bond that passed read-only validation: both operands' +// discoveries, and the issue and logical store that pin the mutation. +type molBondPlan struct { + a, b *molBondDiscovery + targetID string + targetStoreKey string +} + +// Close releases the discoveries' read-only handles. It is idempotent. +func (p *molBondPlan) Close() { + p.a.Close() + p.b.Close() +} + +// planMolBond discovers both operands and validates their store homes, all +// through read-only handles, so a rejected bond never opens a foreign rig +// writable. +func planMolBond(ctx context.Context, localStore storage.DoltStorage, in molBondInput) (*molBondPlan, error) { + discA, err := discoverMolBondOperand(ctx, localStore, in.argA, in.vars) + if err != nil { + return nil, err + } + discB, err := discoverMolBondOperand(ctx, localStore, in.argB, in.vars) + if err != nil { + discA.Close() + return nil, err + } + plan := &molBondPlan{a: discA, b: discB} + plan.targetID, plan.targetStoreKey, err = validateMolBondHomes(localStore, discA, discB) + if err != nil { + plan.Close() + return nil, err + } + return plan, nil +} + +// openMolBondHome opens the bond's validated home for writing: the local store, +// or the routed target reopened writable. A routed home gets the +// external-dependency policy and nothing else, as wireExternalDependencyPolicy +// asks of routed stores: the bond writes into another rig, so the launching +// workspace's hooks and telemetry do not apply to it. An auto-route arrives +// with the policy already applied. +func openMolBondHome(ctx context.Context, localStore storage.DoltStorage, plan *molBondPlan) (storage.DoltStorage, func(), error) { + if plan.targetID == "" || plan.targetStoreKey == dependencyStoreKey(localStore) { + return localStore, func() {}, nil + } + rr, err := resolveAndGetIssueForMutation(ctx, localStore, plan.targetID) + if err != nil { + return nil, nil, err + } + if err := verifyMolBondWritableHome(rr, plan.targetStoreKey); err != nil { + rr.Close() + return nil, nil, err + } + home := rr.Store + if !externaldeps.Applied(home) { + home = wireExternalDependencyPolicy(home) + } + return home, rr.Close, nil +} + +// materializeMolBondOperands re-reads both operands in the bond's home and +// cooks any formula operand in memory. +func materializeMolBondOperands(ctx context.Context, activeStore storage.DoltStorage, plan *molBondPlan, vars map[string]string) (*molBondOperand, *molBondOperand, error) { + opA, err := materializeMolBondOperand(ctx, activeStore, plan.a, vars) + if err != nil { + return nil, nil, err + } + opB, err := materializeMolBondOperand(ctx, activeStore, plan.b, vars) + if err != nil { + return nil, nil, err + } + return opA, opB, nil +} + +// bondMolBondOperands runs the bond operation the two operand types select. +func bondMolBondOperands(ctx context.Context, activeStore storage.DoltStorage, opA, opB *molBondOperand, in molBondInput) (*BondResult, error) { + issueA, issueB := opA.subgraph.Root, opB.subgraph.Root + aIsProto, bIsProto := opA.bondsAsProto(), opB.bondsAsProto() + + // Dispatch based on operand types + // All operations use activeStore; phase flags determine ephemeral vs persistent. + switch { + case aIsProto && bIsProto: + // Compound protos are templates - always persistent + // Note: Proto+proto bonding from formulas is a DB operation, not in-memory + return bondProtoProto(ctx, activeStore, issueA, issueB, in.bondType, in.customTitle, currentActor()) + case aIsProto: + if opA.cooked { + return bondProtoMolWithSubgraph(ctx, activeStore, opA.subgraph, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) + } + return bondProtoMol(ctx, activeStore, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) + case bIsProto: + if opB.cooked { + return bondProtoMolWithSubgraph(ctx, activeStore, opB.subgraph, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) + } + return bondMolProto(ctx, activeStore, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) + default: + return bondMolMol(ctx, activeStore, issueA, issueB, in.bondType, currentActor()) + } } type molBondInput struct { @@ -683,11 +770,169 @@ func minPriority(a, b int) int { return b } +type molBondOperand struct { + subgraph *TemplateSubgraph + cooked bool +} + +// bondsAsProto reports whether the operand bonds as a proto: a template issue, +// or a formula cooked in memory. +func (o *molBondOperand) bondsAsProto() bool { + return o.subgraph.Root.IsTemplate || o.cooked +} + +// molBondDiscovery is the read-only phase of operand resolution. Existing +// issues retain their logical store home; formulas remain storeless. +type molBondDiscovery struct { + operand string + issue *types.Issue + formula string + store storage.DoltStorage + storeKey string + closeFn func() +} + +func (d *molBondDiscovery) Close() { + if d != nil && d.closeFn != nil { + d.closeFn() + d.closeFn = nil + } +} + +// discoverMolBondOperand resolves issue-first through read-only routing. This +// lets store-policy validation finish before any routed database is opened +// writable. Formula fallback remains unconditional: the parser decides whether +// an operand names a formula (#3874). +func discoverMolBondOperand(ctx context.Context, localStore storage.DoltStorage, operand string, vars map[string]string) (*molBondDiscovery, error) { + rr, err := resolveAndGetIssueWithRouting(ctx, localStore, operand) + if err == nil { + return &molBondDiscovery{ + operand: operand, + issue: rr.Issue, + store: rr.Store, + storeKey: dependencyStoreKey(rr.Store), + closeFn: rr.Close, + }, nil + } + parser := formula.NewParser() + f, loadErr := parser.LoadByName(operand) + if loadErr == nil { + // Discovery must fail the same way materialization would, and + // materialization (resolveAndCookFormulaWithVars) resolves inheritance + // before it validates anything: parser.Resolve is what runs + // Formula.Validate, which LoadByName never does (#5761). Validating + // --var values against the resolved formula also checks a var + // declared only in a parent. An enum/pattern/provided-empty violation + // fails the cook, so surfacing it here makes --dry-run and execution + // reject the bond identically, before any writable reopen (#5253). + // Both are wrapped the way the cook wraps them, so the messages match. + resolved, resolveErr := parser.Resolve(f) + if resolveErr != nil { + return nil, fmt.Errorf("resolving formula %q: %w", operand, resolveErr) + } + if err := formula.ValidateProvidedVars(resolved, vars); err != nil { + return nil, fmt.Errorf("formula %q: %w", operand, err) + } + return &molBondDiscovery{operand: operand, formula: resolved.Formula}, nil + } + if !isNotFoundErr(err) { + return nil, err + } + return nil, fmt.Errorf("'%s' not found as issue or formula: %w", operand, loadErr) +} + +// validateMolBondHomes returns the issue and logical store that pin the +// mutation. Formula-only bonds stay local. All checks use read-only handles. +func validateMolBondHomes(localStore storage.DoltStorage, discoveries ...*molBondDiscovery) (string, string, error) { + var targetID, targetKey string + var targetStore storage.DoltStorage + for _, d := range discoveries { + if d == nil || d.issue == nil { + continue + } + if targetStore == nil { + targetID, targetKey, targetStore = d.issue.ID, d.storeKey, d.store + continue + } + if !sameBondStore(targetStore, d.store) { + return "", "", fmt.Errorf("cannot bond %s and %s: they live in different stores/rigs; run the bond from the rig that owns them", targetID, d.issue.ID) + } + } + if targetStore == nil { + return "", dependencyStoreKey(localStore), nil + } + return targetID, targetKey, nil +} + +// verifyMolBondWritableHome closes the TOCTOU gap between read-only discovery +// and writable reopen. Routing or local contents may change between phases; a +// bond must never mutate a logical store that was not the validated home. +func verifyMolBondWritableHome(rr *RoutedResult, expectedStoreKey string) error { + if rr == nil || rr.Store == nil { + return fmt.Errorf("routed bond target reopened without a store") + } + if actual := dependencyStoreKey(rr.Store); actual != expectedStoreKey { + return fmt.Errorf("routed bond target changed between discovery and writable reopen (expected %s, got %s); retry the command", expectedStoreKey, actual) + } + return nil +} + +// sameBondStore reports whether two store handles refer to the same underlying +// database. dependencyStoreKey unwraps decorators before consulting +// StoreLocator, so a HookFiringStore and a separately opened raw route handle +// onto the same database compare equal. +func sameBondStore(a, b storage.DoltStorage) bool { + if a == b { + return true + } + return dependencyStoreKey(a) == dependencyStoreKey(b) +} + +// materializeMolBondOperand is the writable phase. After validation selects one +// store home, existing issues are re-read there and formulas are cooked in +// memory for the bond operation. +func materializeMolBondOperand(ctx context.Context, activeStore storage.DoltStorage, d *molBondDiscovery, vars map[string]string) (*molBondOperand, error) { + if d == nil { + return nil, fmt.Errorf("missing mol bond operand") + } + if d.issue == nil { + subgraph, err := resolveAndCookFormulaWithVars(d.operand, nil, vars) + if err != nil { + if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { + // Don't double-wrap: operand IS a formula that either does not + // validate or was given --var values failing its enum/pattern/ + // required-empty constraints, both distinct from "not found". + return nil, err + } + return nil, fmt.Errorf("'%s' not found as issue or formula: %w", d.operand, err) + } + return &molBondOperand{subgraph: subgraph, cooked: true}, nil + } + + rr, err := resolveAndGetFromStore(ctx, activeStore, d.issue.ID, false) + if err != nil { + return nil, err + } + issue := rr.Issue + if isProto(issue) { + subgraph, err := loadTemplateSubgraph(ctx, activeStore, issue.ID) + if err != nil { + return nil, fmt.Errorf("loading proto subgraph '%s': %w", issue.ID, err) + } + return &molBondOperand{subgraph: subgraph}, nil + } + return &molBondOperand{ + subgraph: &TemplateSubgraph{ + Root: issue, + Issues: []*types.Issue{issue}, + IssueMap: map[string]*types.Issue{issue.ID: issue}, + }, + }, nil +} + // resolveOrDescribe checks if an operand is an issue or formula without cooking. -// Used for dry-run mode. Returns (issue, formulaName, error). -// If it's an issue, issue is set. If it's a formula, formulaName is set. +// Used by proxied-server dry-run mode. Returns (issue, formulaName, error). func resolveOrDescribe(ctx context.Context, s molReader, operand string, vars map[string]string) (*types.Issue, string, error) { - // First, try to resolve as an existing issue id, err := utils.ResolvePartialID(ctx, s, operand) if err == nil { issue, err := s.GetIssue(ctx, id) @@ -696,10 +941,6 @@ func resolveOrDescribe(ctx context.Context, s molReader, operand string, vars ma } } - // Not found as issue — fall through to the formula registry. parser.LoadByName - // returns "formula %q not found in search paths" for genuinely unknown names, - // which is the right error. This matches the resolution behavior of - // bd formula show / bd mol seed / bd mol pour / bd cook. parser := formula.NewParser() f, err := parser.LoadByName(operand) if err != nil { diff --git a/cmd/bd/mol_bond_routing_test.go b/cmd/bd/mol_bond_routing_test.go new file mode 100644 index 000000000..f05a6c54a --- /dev/null +++ b/cmd/bd/mol_bond_routing_test.go @@ -0,0 +1,514 @@ +//go:build cgo + +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// TestDiscoverMolBondOperandRouting is the regression test for the mol-bond +// routing bug (#4714): `bd mol bond ` failed with +// "'' not found as issue or formula" whenever lived in a routed +// store (a prefix-routed rig, or the contributor planning store), even though +// `bd show ` resolved it. mol bond resolved operands against the local +// store only and never fell through to the routing fallback that +// show/update/close use. +// +// discoverMolBondOperand resolves through the same local, prefix-route, and +// auto-route order, read-only, before the accepted home is reopened writable. +// +// The cases mirror TestResolveCloseTargets: +// - maintainer-mode local resolution (no routing involved) +// - an operand that lives only in the routed planning store, reached through +// the contributor, maintainer, or default auto-route; each reopens its +// home writable for the bond +func TestDiscoverMolBondOperandRouting(t *testing.T) { + cases := []struct { + name string + role string + enableRouting bool + routingKey string + localSeed string + planningSeed string + operand string + wantWritable bool + wantStore string + }{ + { + name: "maintainer_local_issue", + role: "maintainer", + localSeed: "shared-local", + operand: "shared-local", + wantStore: "local", + }, + { + name: "contributor_routed_issue", + role: "contributor", + enableRouting: true, + routingKey: "routing.contributor", + planningSeed: "shared-routed", + operand: "shared-routed", + wantWritable: true, + wantStore: "planning", + }, + { + name: "maintainer_routed_issue", + role: "maintainer", + enableRouting: true, + routingKey: "routing.maintainer", + planningSeed: "shared-maintainer", + operand: "shared-maintainer", + wantWritable: true, + wantStore: "planning", + }, + { + name: "default_routed_issue", + role: "maintainer", + enableRouting: true, + routingKey: "routing.default", + planningSeed: "shared-default", + operand: "shared-default", + wantWritable: true, + wantStore: "planning", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + initConfigForTest(t) + neutralizeRoutingModeEnv(t) + + tmpDir := t.TempDir() + repoDir := filepath.Join(tmpDir, "repo") + planningDir := filepath.Join(tmpDir, "planning") + + runCmd(t, tmpDir, "git", "init", repoDir) + // Force repo-local hooks so tests ignore any global hooksPath override. + runCmd(t, repoDir, "git", "config", "core.hooksPath", ".git/hooks") + runCmd(t, repoDir, "git", "config", "beads.role", tc.role) + + primaryStore := newTestStoreIsolatedDB(t, filepath.Join(repoDir, ".beads", "beads.db"), "shared") + ctx := context.Background() + + if tc.enableRouting { + if err := primaryStore.SetConfig(ctx, "routing.mode", "auto"); err != nil { + t.Fatalf("set routing.mode: %v", err) + } + if err := primaryStore.SetConfig(ctx, tc.routingKey, planningDir); err != nil { + t.Fatalf("set %s: %v", tc.routingKey, err) + } + } + + if tc.localSeed != "" { + seedIssue(t, ctx, primaryStore, tc.localSeed) + } + if tc.planningSeed != "" { + planningStore := newTestStoreIsolatedDB(t, filepath.Join(planningDir, ".beads", "beads.db"), "shared") + seedIssue(t, ctx, planningStore, tc.planningSeed) + if err := planningStore.Close(); err != nil { + t.Fatalf("close planning store: %v", err) + } + } + + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + if err := os.Chdir(repoDir); err != nil { + t.Fatalf("chdir repoDir: %v", err) + } + // Role detection reads the process-wide RepoContext cache, which the + // previous case built for its own, now-deleted repo. Rebuild it from + // repoDir. + resetRepoContextCachesForTest(t) + + disc, err := discoverMolBondOperand(ctx, primaryStore, tc.operand, nil) + if err != nil { + t.Fatalf("discoverMolBondOperand(%q): %v", tc.operand, err) + } + defer disc.Close() + if disc.issue == nil || disc.issue.ID != tc.operand || disc.formula != "" { + t.Fatalf("operand %q discovery = %#v", tc.operand, disc) + } + localKey := dependencyStoreKey(primaryStore) + if tc.wantStore == "local" && disc.storeKey != localKey { + t.Errorf("operand %q storeKey = %q, want local %q", tc.operand, disc.storeKey, localKey) + } + if tc.wantStore == "planning" && disc.storeKey == localKey { + t.Errorf("operand %q unexpectedly resolved to local store", tc.operand) + } + if tc.wantWritable { + disc.Close() + rr, err := resolveAndGetIssueForMutation(ctx, primaryStore, tc.operand) + if err != nil { + t.Fatalf("resolveAndGetIssueForMutation(%q): %v", tc.operand, err) + } + defer rr.Close() + requireWritableAutoRoute(t, "operand "+tc.operand, rr) + } + }) + } +} + +// TestDiscoverMolBondOperandFormulaWinsAmbiguousIssuePrefix preserves the +// issue-first contract without making an ambiguous issue prefix shadow a valid +// formula of the same name. The issue lookup error is actionable only when the +// formula parser also rejects the operand. +func TestDiscoverMolBondOperandFormulaWinsAmbiguousIssuePrefix(t *testing.T) { + initConfigForTest(t) + ctx := context.Background() + + repoDir := t.TempDir() + beadsDir := filepath.Join(repoDir, ".beads") + localStore := newTestStoreIsolatedDB(t, filepath.Join(beadsDir, "beads.db"), "amb") + seedIssue(t, ctx, localStore, "ambiguous-formula-one") + seedIssue(t, ctx, localStore, "ambiguous-formula-two") + + formulaDir := filepath.Join(beadsDir, "formulas") + if err := os.MkdirAll(formulaDir, 0o755); err != nil { + t.Fatalf("create formula directory: %v", err) + } + const name = "ambiguous-formula" + formula := []byte(`formula = "ambiguous-formula" +version = 1 +type = "workflow" + +[[steps]] +id = "step" +title = "Step" +type = "task" +`) + if err := os.WriteFile(filepath.Join(formulaDir, name+".formula.toml"), formula, 0o644); err != nil { + t.Fatalf("write formula: %v", err) + } + + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + if err := os.Chdir(repoDir); err != nil { + t.Fatalf("chdir repoDir: %v", err) + } + + disc, err := discoverMolBondOperand(ctx, localStore, name, nil) + if err != nil { + t.Fatalf("discoverMolBondOperand(%q): %v", name, err) + } + defer disc.Close() + if disc.issue != nil || disc.formula != name { + t.Fatalf("ambiguous issue prefix should fall back to formula: %#v", disc) + } +} + +// TestMolBondPrefixRoutedWriteThrough verifies the bond MUTATION against a +// prefix-routed rig store, not just operand resolution: both operands live in +// a routed rig, read-only validation accepts the same-rig pair, the selected +// store reopens writable, and the dependency is visible +// afterward through a fresh routed read — proving the write committed in the +// routed database rather than the local one. +// +// NOTE: This test uses os.Chdir and cannot run in parallel with other tests. +func TestMolBondPrefixRoutedWriteThrough(t *testing.T) { + initConfigForTest(t) + ctx := context.Background() + + tmpDir := t.TempDir() + townBeadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(townBeadsDir, 0755); err != nil { + t.Fatalf("create town beads dir: %v", err) + } + rigBeadsDir := filepath.Join(tmpDir, "rig", ".beads") + if err := os.MkdirAll(rigBeadsDir, 0755); err != nil { + t.Fatalf("create rig beads dir: %v", err) + } + + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + + rigDBPath := filepath.Join(rigBeadsDir, "dolt") + rigStore := newTestStoreIsolatedDB(t, rigDBPath, "gt") + seedIssue(t, ctx, rigStore, "gt-mol1") + seedIssue(t, ctx, rigStore, "gt-mol2") + // Release the rig store before routing reopens it. + rigStore.Close() + + otherRigBeadsDir := filepath.Join(tmpDir, "other-rig", ".beads") + if err := os.MkdirAll(otherRigBeadsDir, 0755); err != nil { + t.Fatalf("create other rig beads dir: %v", err) + } + otherStore := newTestStoreIsolatedDB(t, filepath.Join(otherRigBeadsDir, "dolt"), "zz") + seedIssue(t, ctx, otherStore, "zz-other") + otherStore.Close() + + routesPath := filepath.Join(townBeadsDir, "routes.jsonl") + routes := []byte("{\"prefix\":\"gt-\",\"path\":\"rig\"}\n{\"prefix\":\"zz-\",\"path\":\"other-rig\"}\n") + if err := os.WriteFile(routesPath, routes, 0644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + + oldDbPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDbPath }) + + oldWd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(tmpDir); err != nil { + t.Fatalf("chdir: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWd) }) + + discA, err := discoverMolBondOperand(ctx, townStore, "gt-mol1", nil) + if err != nil { + t.Fatalf("discover first routed operand: %v", err) + } + defer discA.Close() + discB, err := discoverMolBondOperand(ctx, townStore, "gt-mol2", nil) + if err != nil { + t.Fatalf("discover second routed operand: %v", err) + } + defer discB.Close() + targetID, targetKey, err := validateMolBondHomes(townStore, discA, discB) + if err != nil { + t.Fatalf("validate same-rig operands: %v", err) + } + discA.Close() + discB.Close() + + routed, err := resolveAndGetIssueForMutation(ctx, townStore, targetID) + if err != nil { + t.Fatalf("open accepted target writable: %v", err) + } + defer routed.Close() + if got := dependencyStoreKey(routed.Store); got != targetKey { + t.Fatalf("writable reopen changed store: got %q, want %q", got, targetKey) + } + opA, err := materializeMolBondOperand(ctx, routed.Store, discA, nil) + if err != nil { + t.Fatalf("materialize first operand: %v", err) + } + opB, err := materializeMolBondOperand(ctx, routed.Store, discB, nil) + if err != nil { + t.Fatalf("materialize second operand: %v", err) + } + if _, err := bondMolMol(ctx, routed.Store, opA.subgraph.Root, opB.subgraph.Root, types.BondTypeSequential, "test"); err != nil { + t.Fatalf("bondMolMol against the routed store: %v", err) + } + routed.Close() + + // Verify through a fresh routed read that the bond committed in the rig + // database: gt-mol2 (B) now depends on gt-mol1 (A). + rr, err := resolveViaPrefixRoutingWithAccess(ctx, "gt-mol2", false, false) + if err != nil { + t.Fatalf("reopening routed store to verify the write: %v", err) + } + defer rr.Close() + deps, err := rr.Store.GetDependenciesWithMetadata(ctx, "gt-mol2") + if err != nil { + t.Fatalf("reading dependencies from the routed store: %v", err) + } + found := false + for _, dep := range deps { + if dep.ID == "gt-mol1" && dep.DependencyType == types.DepBlocks { + found = true + } + } + if !found { + t.Fatalf("bond did not commit in the routed store: gt-mol2 dependencies = %+v", deps) + } + rr.Close() + + crossA, err := discoverMolBondOperand(ctx, townStore, "gt-mol1", nil) + if err != nil { + t.Fatalf("discover cross-store operand A: %v", err) + } + defer crossA.Close() + crossB, err := discoverMolBondOperand(ctx, townStore, "zz-other", nil) + if err != nil { + t.Fatalf("discover cross-store operand B: %v", err) + } + defer crossB.Close() + if _, _, err := validateMolBondHomes(townStore, crossA, crossB); err == nil || !strings.Contains(err.Error(), "different stores/rigs") { + t.Fatalf("cross-store pair error = %v, want explicit rejection", err) + } + crossA.Close() + crossB.Close() + + routes = append(routes, []byte("{\"prefix\":\"bad-\",\"path\":\"missing-rig\"}\n")...) + if err := os.WriteFile(routesPath, routes, 0644); err != nil { + t.Fatalf("write matched failure route: %v", err) + } + if _, err := discoverMolBondOperand(ctx, townStore, "bad-target", nil); err == nil || !strings.Contains(err.Error(), "no dolt_database") { + t.Fatalf("matched-route failure = %v, want actionable target metadata error", err) + } + + drift, err := discoverMolBondOperand(ctx, townStore, "zz-other", nil) + if err != nil { + t.Fatalf("discover drift fixture: %v", err) + } + _, driftKey, err := validateMolBondHomes(townStore, drift) + if err != nil { + t.Fatalf("validate drift fixture: %v", err) + } + drift.Close() + seedIssue(t, ctx, townStore, "zz-other") + reopened, err := resolveAndGetIssueForMutation(ctx, townStore, "zz-other") + if err != nil { + t.Fatalf("reopen drift fixture: %v", err) + } + defer reopened.Close() + if err := verifyMolBondWritableHome(reopened, driftKey); err == nil || !strings.Contains(err.Error(), "changed between discovery") { + t.Fatalf("reopen identity mismatch = %v, want retryable safety error", err) + } +} + +// TestSameBondStoreSeparateHandles is the regression test for the store +// identity check: two separately-opened handles onto the same routed database +// (each route-open constructs a fresh store in server mode) must compare as +// the same store, not be rejected as a cross-store bond. +// +// NOTE: This test uses os.Chdir and cannot run in parallel with other tests. +func TestSameBondStoreSeparateHandles(t *testing.T) { + initConfigForTest(t) + ctx := context.Background() + + tmpDir := t.TempDir() + townBeadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(townBeadsDir, 0755); err != nil { + t.Fatalf("create town beads dir: %v", err) + } + rigBeadsDir := filepath.Join(tmpDir, "rig", ".beads") + if err := os.MkdirAll(rigBeadsDir, 0755); err != nil { + t.Fatalf("create rig beads dir: %v", err) + } + + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + + rigDBPath := filepath.Join(rigBeadsDir, "dolt") + rigStore := newTestStoreIsolatedDB(t, rigDBPath, "gt") + seedIssue(t, ctx, rigStore, "gt-idn1") + rigStore.Close() + + routesPath := filepath.Join(townBeadsDir, "routes.jsonl") + if err := os.WriteFile(routesPath, []byte(`{"prefix":"gt-","path":"rig"}`), 0644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + + oldDbPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDbPath }) + + oldWd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(tmpDir); err != nil { + t.Fatalf("chdir: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWd) }) + + rrA, err := resolveViaPrefixRoutingWithAccess(ctx, "gt-idn1", true, false) + if err != nil { + t.Fatalf("first route-open: %v", err) + } + defer rrA.Close() + rrB, err := resolveViaPrefixRoutingWithAccess(ctx, "gt-idn1", true, false) + if err != nil { + t.Fatalf("second route-open: %v", err) + } + defer rrB.Close() + + if rrA.Store == rrB.Store { + t.Fatal("test precondition: expected two distinct store handles from separate route-opens") + } + if !sameBondStore(rrA.Store, rrB.Store) { + t.Fatal("sameBondStore: two handles onto the same routed database must compare as the same store") + } + + // The negative case matters just as much: genuinely different databases + // must NOT compare as the same store, or validation would silently + // allow the cross-store bonds it exists to reject. + if sameBondStore(townStore, rrA.Store) { + t.Fatal("sameBondStore: handles onto different databases must not compare as the same store") + } + + wrapped := storage.NewHookFiringStore(rrA.Store, nil) + if !sameBondStore(wrapped, rrB.Store) { + t.Fatal("sameBondStore: decorator-wrapped and raw handles onto the same database must compare equal") + } +} + +// TestMolBondContributorAutoRoutedWriteThrough runs bd mol bond's pipeline +// (planMolBond, openMolBondHome, materialize, bond) on two issues that live in a +// contributor's planning repository, reached through the contributor +// auto-route. The planning repository is the contributor's own, so the bond +// commits there as an update or close of a planning issue does +// (TestContributorAutoRouteWritesThrough): the validated home reopens writable +// and a fresh routed read sees the dependency. +func TestMolBondContributorAutoRoutedWriteThrough(t *testing.T) { + primaryStore, _, _ := setupAutoRoute(t, "contributor", "routing.contributor", "shared-bond") + ctx := context.Background() + + in := molBondInput{argA: "shared-hub", argB: "shared-bond", bondType: types.BondTypeSequential} + plan, err := planMolBond(ctx, primaryStore, in) + if err != nil { + t.Fatalf("planMolBond on two contributor-routed issues: %v", err) + } + defer plan.Close() + if plan.targetStoreKey == dependencyStoreKey(primaryStore) { + t.Fatalf("bond home resolved to the local store; want the contributor planning store") + } + plan.Close() + + home, closeHome, err := openMolBondHome(ctx, primaryStore, plan) + if err != nil { + t.Fatalf("openMolBondHome: %v", err) + } + defer closeHome() + if routedStoreIsReadOnly(t, home) { + t.Fatalf("the contributor auto-route reopened the bond home read-only") + } + opA, opB, err := materializeMolBondOperands(ctx, home, plan, nil) + if err != nil { + t.Fatalf("materialize operands: %v", err) + } + if _, err := bondMolBondOperands(ctx, home, opA, opB, in); err != nil { + t.Fatalf("bond in the contributor planning store: %v", err) + } + closeHome() + + // Verify through a fresh routed read that the bond committed in the + // planning database: shared-bond (B) now depends on shared-hub (A). + rr, err := resolveAndGetIssueWithRouting(ctx, primaryStore, "shared-bond") + if err != nil { + t.Fatalf("re-reading the routed issue: %v", err) + } + defer rr.Close() + if !rr.Routed { + t.Fatalf("re-read resolved shared-bond without routing; want the contributor auto-route") + } + deps, err := rr.Store.GetDependenciesWithMetadata(ctx, "shared-bond") + if err != nil { + t.Fatalf("reading dependencies from the planning store: %v", err) + } + found := false + for _, dep := range deps { + if dep.ID == "shared-hub" && dep.DependencyType == types.DepBlocks { + found = true + } + } + if !found { + t.Fatalf("bond did not commit in the planning store: shared-bond dependencies = %+v", deps) + } +} diff --git a/cmd/bd/mol_bond_var_validation_test.go b/cmd/bd/mol_bond_var_validation_test.go new file mode 100644 index 000000000..acc6b5172 --- /dev/null +++ b/cmd/bd/mol_bond_var_validation_test.go @@ -0,0 +1,198 @@ +//go:build cgo + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/formula" +) + +// Regression coverage for the #5253 guarantees on the routed bond path: +// bd mol bond resolves operands through discoverMolBondOperand (read-only +// routing discovery) rather than resolveOrDescribe, so the discovery fallback +// must enforce the same --var enum/pattern/provided-empty validation — for +// --dry-run and execution alike — and a var-validation failure on a real +// formula must surface directly instead of being masked as +// "'X' not found as issue or formula". + +func makeMolBondVarTestCmd(dryRun bool, varFlags []string) *cobra.Command { + c := &cobra.Command{Use: "bond"} + c.Flags().String("type", "sequential", "") + c.Flags().String("as", "", "") + c.Flags().Bool("dry-run", dryRun, "") + c.Flags().Bool("ephemeral", false, "") + c.Flags().Bool("pour", false, "") + c.Flags().String("ref", "", "") + c.Flags().StringArray("var", varFlags, "") + return c +} + +func runMolBondVarViolation(t *testing.T, dryRun bool) (error, string) { + t.Helper() + writeVarValidationFormula(t) + s := newTestStoreWithPrefix(t, filepath.Join(t.TempDir(), "test.db"), "test") + withWispTestGlobals(t, s, context.Background()) + + var runErr error + stderr := captureStderr(t, func() { + runErr = runMolBond(makeMolBondVarTestCmd(dryRun, []string{"policy=bogus", "slug=abc"}), + []string{"pour-wisp-var-validation-test", "pour-wisp-var-validation-test"}) + }) + return runErr, stderr +} + +// TestMolBondDryRunRejectsEnumViolatingVar pins #4714's framing: the dry-run +// must fail the same way the real bond would, not preview success for a bond +// execution refuses. +func TestMolBondDryRunRejectsEnumViolatingVar(t *testing.T) { + runErr, stderr := runMolBondVarViolation(t, true) + + if runErr == nil { + t.Fatal("mol bond --dry-run accepted a --var value outside the declared enum") + } + if !strings.Contains(stderr, "not in allowed values") { + t.Fatalf("stderr = %q, want an enum-violation message", stderr) + } + if strings.Contains(stderr, "not found as issue or formula") { + t.Fatalf("stderr = %q: the var-validation failure was masked as not-found", stderr) + } +} + +func TestMolBondExecutionRejectsEnumViolatingVar(t *testing.T) { + runErr, stderr := runMolBondVarViolation(t, false) + + if runErr == nil { + t.Fatal("mol bond accepted a --var value outside the declared enum") + } + if !strings.Contains(stderr, "not in allowed values") { + t.Fatalf("stderr = %q, want an enum-violation message", stderr) + } + if strings.Contains(stderr, "not found as issue or formula") { + t.Fatalf("stderr = %q: the var-validation failure was masked as not-found", stderr) + } +} + +// TestMaterializeMolBondOperandPassesVarValidationThrough covers the writable +// phase directly: even if a violating value reaches the cook (discovery only +// checks vars that are provided against the formula it resolved), the sentinel +// must pass through unwrapped so callers and users see the constraint +// violation, not a bogus not-found. +func TestMaterializeMolBondOperandPassesVarValidationThrough(t *testing.T) { + writeVarValidationFormula(t) + s := newTestStoreWithPrefix(t, filepath.Join(t.TempDir(), "test.db"), "test") + + _, err := materializeMolBondOperand(context.Background(), s, + &molBondDiscovery{operand: "pour-wisp-var-validation-test"}, + map[string]string{"policy": "bogus", "slug": "abc"}) + if err == nil { + t.Fatal("materializeMolBondOperand cooked a formula whose --var values violate its enum") + } + if !errors.Is(err, formula.ErrVarValidation) { + t.Fatalf("err = %v, want errors.Is(err, formula.ErrVarValidation)", err) + } + if strings.Contains(err.Error(), "not found as issue or formula") { + t.Fatalf("err = %q: the var-validation failure was double-wrapped as not-found", err) + } +} + +// The direct (non-proxied) bond previews and executes through +// discoverMolBondOperand and materializeMolBondOperand, not resolveOrDescribe +// and resolveOrCookToSubgraph, so #5761's dry-run guarantee has to hold for +// this pair too: discovery is all a --dry-run runs, so whatever it accepts is +// previewed as a successful bond. As in mol_bond_dryrun_validation_test.go, the +// two phases are asserted against EACH OTHER, so a validation rule added later +// cannot re-open the gap. +func TestDiscoverMolBondOperandRejectsWhatMaterializeRejects(t *testing.T) { + writeSearchableFormula(t, "dryrun-invalid-gate", invalidGateFormula) + s := newTestStoreWithPrefix(t, filepath.Join(t.TempDir(), "test.db"), "test") + ctx := context.Background() + + _, discoverErr := discoverMolBondOperand(ctx, s, "dryrun-invalid-gate", nil) + _, materializeErr := materializeMolBondOperand(ctx, s, + &molBondDiscovery{operand: "dryrun-invalid-gate"}, nil) + + if materializeErr == nil { + t.Fatal("materialization accepted an invalid formula, so this test is no longer testing the gap") + } + if discoverErr == nil { + t.Fatalf("discovery accepted a formula materialization rejects (materialization said: %v)", materializeErr) + } + for name, err := range map[string]error{"discovery": discoverErr, "materialization": materializeErr} { + if !errors.Is(err, formula.ErrValidation) { + t.Errorf("%s error does not wrap formula.ErrValidation: %v", name, err) + } + if !strings.Contains(err.Error(), "waits_for") { + t.Errorf("%s error does not name the failing field: %v", name, err) + } + if strings.Contains(err.Error(), "not found as issue or formula") { + t.Errorf("%s reported an invalid formula as not found: %v", name, err) + } + } + if discoverErr.Error() != materializeErr.Error() { + t.Errorf("discovery and materialization report different errors:\n discovery: %v\n materialization: %v", discoverErr, materializeErr) + } +} + +// The cook validates --var values against the MERGED formula, so discovery +// must resolve inheritance first: a var declared only in a parent is otherwise +// unchecked, and the dry-run previews a bond execution refuses. +func TestDiscoverMolBondOperandValidatesVarsAgainstTheMergedFormula(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, ".beads", "formulas") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir formulas dir: %v", err) + } + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name+".formula.toml"), []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + write("discover-parent", `formula = "discover-parent" +version = 1 +type = "workflow" + +[vars.env] +enum = ["dev", "prod"] + +[[steps]] +id = "deploy" +title = "Deploy to {{env}}" +type = "task" +`) + write("discover-child", `formula = "discover-child" +version = 1 +type = "workflow" +extends = ["discover-parent"] +`) + t.Setenv("GT_ROOT", root) + + s := newTestStoreWithPrefix(t, filepath.Join(t.TempDir(), "test.db"), "test") + ctx := context.Background() + vars := map[string]string{"env": "staging"} // not in the parent's enum + + _, discoverErr := discoverMolBondOperand(ctx, s, "discover-child", vars) + _, materializeErr := materializeMolBondOperand(ctx, s, + &molBondDiscovery{operand: "discover-child"}, vars) + + if materializeErr == nil { + t.Fatal("materialization accepted a var value outside the inherited enum") + } + if discoverErr == nil { + t.Fatalf("discovery accepted a var value materialization rejects; the enum is declared only in the parent (materialization said: %v)", materializeErr) + } + for name, err := range map[string]error{"discovery": discoverErr, "materialization": materializeErr} { + if !errors.Is(err, formula.ErrVarValidation) { + t.Errorf("%s error does not wrap formula.ErrVarValidation: %v", name, err) + } + } + if discoverErr.Error() != materializeErr.Error() { + t.Errorf("discovery and materialization report different errors:\n discovery: %v\n materialization: %v", discoverErr, materializeErr) + } +} diff --git a/cmd/bd/routed.go b/cmd/bd/routed.go index cc1d4d58e..4221bcf93 100644 --- a/cmd/bd/routed.go +++ b/cmd/bd/routed.go @@ -6,17 +6,22 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" ) +var errRoutingUnavailable = errors.New("routing unavailable") + // isNotFoundErr returns true if the error indicates the issue was not found. // This covers both storage.ErrNotFound (from GetIssue) and the plain error // from ResolvePartialID which doesn't wrap the sentinel. @@ -30,6 +35,15 @@ func isNotFoundErr(err error) bool { return false } +// isRoutingFailure reports whether a routed tier's error must reach the caller +// instead of falling through to the next tier: anything but "this tier has no +// route" (errRoutingUnavailable) or a plain not-found. An unreadable +// routes.jsonl or a routed store that fails to open is why the lookup missed, +// and returning the local not-found in its place would misstate the cause. +func isRoutingFailure(err error) bool { + return err != nil && !errors.Is(err, errRoutingUnavailable) && !isNotFoundErr(err) +} + // RoutedResult contains the result of a routed issue lookup type RoutedResult struct { Issue *types.Issue @@ -43,31 +57,51 @@ type RoutedResult struct { func (r *RoutedResult) Close() { if r.closeFn != nil { r.closeFn() + r.closeFn = nil } } +// commitRoutedWrite commits what command wrote through result when result is +// routed. The post-run auto-commit covers only the local store, so a command +// that writes to a routed target commits that write itself, as bd update and +// bd close do. An unrouted result is left to the post-run auto-commit. +func commitRoutedWrite(ctx context.Context, result *RoutedResult, command string) error { + if !result.Routed { + return nil + } + return commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ + Command: command, + IssueIDs: []string{result.ResolvedID}, + }) +} + // resolveAndGetIssueWithRouting resolves a partial ID and gets the issue. // Tries the local store first, then prefix-based routing via routes.jsonl, -// then falls back to contributor auto-routing. +// then falls back to auto-routing (routing.mode=auto). A routing failure, such +// as an unreadable routes.jsonl or a routed store that fails to open, is +// returned as itself rather than as the local not-found. // // Returns a RoutedResult containing the issue, resolved ID, and the store to use. // The caller MUST call result.Close() when done to release any routed storage. // -// Prefix-routed target stores are opened read-only; mutating commands must use -// resolveAndGetIssueForMutation instead so a routed read can never write -// migrations or other open-time mutations into a foreign project (GH#3231, #4141). +// Routed target stores, prefix and auto alike, are opened read-only; mutating +// commands must use resolveAndGetIssueForMutation instead so a routed read can +// never write migrations or other open-time mutations into a foreign project +// (GH#3231, #4141). func resolveAndGetIssueWithRouting(ctx context.Context, localStore storage.DoltStorage, id string) (*RoutedResult, error) { return resolveAndGetIssueWithRoutingAccess(ctx, localStore, id, false) } // resolveAndGetIssueForMutation resolves an issue like -// resolveAndGetIssueWithRouting, but opens prefix-routed target stores in -// writable mode so mutation commands can commit to the routed repository. +// resolveAndGetIssueWithRouting, but opens the routed target store writable so +// mutation commands commit to the repository that holds the issue: a +// prefix-routed rig, or the target of any auto-route, a contributor's planning +// repository included (openAutoRoutedStore). func resolveAndGetIssueForMutation(ctx context.Context, localStore storage.DoltStorage, id string) (*RoutedResult, error) { return resolveAndGetIssueWithRoutingAccess(ctx, localStore, id, true) } -func resolveAndGetIssueWithRoutingAccess(ctx context.Context, localStore storage.DoltStorage, id string, writablePrefixRoute bool) (*RoutedResult, error) { +func resolveAndGetIssueWithRoutingAccess(ctx context.Context, localStore storage.DoltStorage, id string, writable bool) (*RoutedResult, error) { // Try local store first. result, err := resolveAndGetFromStore(ctx, localStore, id, false) if err == nil { @@ -78,17 +112,20 @@ func resolveAndGetIssueWithRoutingAccess(ctx context.Context, localStore storage // This handles cross-rig lookups where the ID's prefix maps to a different // database (e.g., hr-8wn.1 routes to the herald rig's database). if isNotFoundErr(err) { - if prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, writablePrefixRoute, false); prefixErr == nil { + if prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, writable, false); prefixErr == nil { return prefixResult, nil + } else if isRoutingFailure(prefixErr) { + return nil, prefixErr } } - // If not found via prefix routing, try contributor auto-routing as fallback (GH#2345). - // Auto-routed stores stay read-only even for write-intent callers (writablePrefixRoute): - // this path hydrates foreign contributor projects, which must never be mutated. + // If not found via prefix routing, try auto-routing as fallback (GH#2345). + // The target reopens writable when the caller has write intent. if isNotFoundErr(err) { - if autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id, false); autoErr == nil { + if autoResult, autoErr := resolveViaAutoRoutingWithAccess(ctx, localStore, id, writable, false); autoErr == nil { return autoResult, nil + } else if isRoutingFailure(autoErr) { + return nil, autoErr } } @@ -162,19 +199,23 @@ func resolveAndGetFromStoreExact(ctx context.Context, s storage.DoltStorage, id // resolveAndGetIssueForMutationExact resolves an issue like // resolveAndGetIssueForMutation, but requires an EXACT id match against each // store it tries (no leading-prefix abbreviation matching) — see -// utils.ResolvePartialIDExact. Cross-rig prefix routing and contributor -// auto-routing are still attempted as fallbacks exactly as before if the -// local store doesn't have the issue at all; only the per-store resolution -// within each tier is tightened. Used by write paths where an id argument +// utils.ResolvePartialIDExact. Cross-rig prefix routing and auto-routing are +// still attempted as fallbacks exactly as before if the local store doesn't +// have the issue at all; only the per-store resolution within each tier is +// tightened. Used by write paths where an id argument // that doesn't exactly name a real issue must error out instead of silently // mutating whatever it fuzzy-matches (`bd comment list `, a typo for // `bd comments list`, was silently resolving "list" to a wisp whose hash -// happened to start with "list"). +// happened to start with "list"). A routing failure is returned as itself, as +// resolveAndGetIssueForMutation returns it. func resolveAndGetIssueForMutationExact(ctx context.Context, localStore storage.DoltStorage, id string) (*RoutedResult, error) { result, err := resolveAndGetFromStoreExact(ctx, localStore, id, false) if err == nil { return result, nil } + if !isNotFoundErr(err) { + return nil, err + } // The routed tiers resolve with utils.ResolvePartialIDExact too, so each // can mint utils.ErrAbbreviatedIDNotAllowed of its own — the abbreviation @@ -182,47 +223,53 @@ func resolveAndGetIssueForMutationExact(ctx context.Context, localStore storage. // wrappers hand that error back unwrapped. Keep the first such refusal: // the local error is only a plain not-found, and returning it would tell // the caller no issue matches an id a routed store demonstrably holds, - // which is exactly the falsehood the sentinel exists to prevent. + // which is exactly the falsehood the sentinel exists to prevent. The + // refusal is checked before isRoutingFailure, which it would also satisfy, + // so the auto tier still gets its chance at an exact match. var abbrevErr error - - if isNotFoundErr(err) { - prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, true, true) - if prefixErr == nil { - return prefixResult, nil - } - if errors.Is(prefixErr, utils.ErrAbbreviatedIDNotAllowed) { - abbrevErr = prefixErr - } + prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, true, true) + switch { + case prefixErr == nil: + return prefixResult, nil + case errors.Is(prefixErr, utils.ErrAbbreviatedIDNotAllowed): + abbrevErr = prefixErr + case isRoutingFailure(prefixErr): + return nil, prefixErr } - if isNotFoundErr(err) { - autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id, true) - if autoErr == nil { - return autoResult, nil - } - if abbrevErr == nil && errors.Is(autoErr, utils.ErrAbbreviatedIDNotAllowed) { - abbrevErr = autoErr - } - } - - if abbrevErr != nil { + // Write intent, as in resolveAndGetIssueForMutation: the auto-route opens + // writable so the write commits where the issue lives. The auto tier's own + // abbreviation refusal is a routing failure here, and is returned as one. + autoResult, autoErr := resolveViaAutoRoutingWithAccess(ctx, localStore, id, true, true) + switch { + case autoErr == nil: + return autoResult, nil + case abbrevErr != nil: return nil, abbrevErr + case isRoutingFailure(autoErr): + return nil, autoErr } return nil, err } -// resolveViaAutoRouting attempts to find an issue using contributor auto-routing. +// resolveViaAutoRoutingWithAccess attempts to find an issue using auto-routing. // This is the fallback when the local store doesn't have the issue (GH#2345). // Returns a RoutedResult if the issue is found in the auto-routed store. // +// writable is the caller's write intent: the target opens writable for it, +// whichever rule selected the target (openAutoRoutedStore). +// // exact selects utils.ResolvePartialIDExact over the default abbreviation- // tolerant resolver for the auto-routed store lookup, mirroring the local-store // distinction in resolveAndGetFromStore vs resolveAndGetFromStoreExact. -func resolveViaAutoRouting(ctx context.Context, localStore storage.DoltStorage, id string, exact bool) (*RoutedResult, error) { - routedStore, routed, _, err := openRoutedReadStore(ctx, localStore) - if err != nil || !routed { - return nil, fmt.Errorf("no auto-routed store available") +func resolveViaAutoRoutingWithAccess(ctx context.Context, localStore storage.DoltStorage, id string, writable bool, exact bool) (*RoutedResult, error) { + routedStore, routed, _, err := openAutoRoutedStore(ctx, localStore, writable) + if err != nil { + return nil, err + } + if !routed { + return nil, fmt.Errorf("%w: no auto-routed store available", errRoutingUnavailable) } var result *RoutedResult @@ -267,69 +314,25 @@ func resolveViaPrefixRouting(ctx context.Context, id string) (*RoutedResult, err // the routed store lookup, mirroring resolveAndGetFromStore vs // resolveAndGetFromStoreExact. func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable bool, exact bool) (*RoutedResult, error) { - if strings.Index(id, "-") <= 0 { - return nil, fmt.Errorf("no prefix in ID %q", id) + matchedRoute, targetBeadsDir, err := lookupPrefixRoute(id) + if err != nil { + return nil, err } - // Find the resolved beads directory (where routes.jsonl lives) - currentBeadsDir := resolveCommandBeadsDir(dbPath) - if currentBeadsDir == "" { - return nil, fmt.Errorf("no beads directory available") - } - - // Load routes from routes.jsonl - routes, err := loadPrefixRoutes(currentBeadsDir) - if err != nil || len(routes) == 0 { - return nil, fmt.Errorf("no routes available") - } - - // Find the most specific route for this ID - matchedRoute := matchPrefixRoute(routes, id) - if matchedRoute == nil { - return nil, fmt.Errorf("no route for ID %q", id) - } - prefix := matchedRoute.Prefix - - // Skip if the route points to current directory (town-level, already checked) - if matchedRoute.Path == "." { - return nil, fmt.Errorf("route points to current database") - } - - // Derive the town root from the current beads dir. - // currentBeadsDir is typically /.beads - townRoot := filepath.Dir(currentBeadsDir) - - // Resolve the target rig's .beads directory - rigDir := filepath.Join(townRoot, matchedRoute.Path) - targetBeadsDir := beads.FollowRedirect(filepath.Join(rigDir, ".beads")) - // Check that the target has a different dolt_database - targetDB := readDoltDatabase(targetBeadsDir) - if targetDB == "" { - return nil, fmt.Errorf("target rig has no dolt_database configured") + targetDB, err := loadDoltDatabase(targetBeadsDir) + if err == nil && targetDB == "" { + err = fmt.Errorf("%s has no dolt_database field", filepath.Join(targetBeadsDir, "metadata.json")) + } + if err != nil { + return nil, fmt.Errorf("prefix route %q to %s: target rig has no dolt_database configured: %w", matchedRoute.Prefix, matchedRoute.Path, err) } - debug.Logf("[routing] Prefix %q matched route to %s (database: %s)\n", prefix, matchedRoute.Path, targetDB) + debug.Logf("[routing] Prefix %q matched route to %s (database: %s)\n", matchedRoute.Prefix, matchedRoute.Path, targetDB) - // We need to temporarily override BEADS_DOLT_SERVER_DATABASE so server-mode - // stores connect to the correct database on the shared Dolt server. - origDB := os.Getenv("BEADS_DOLT_SERVER_DATABASE") - _ = os.Setenv("BEADS_DOLT_SERVER_DATABASE", targetDB) - var targetStore storage.DoltStorage - var openErr error - if writable { - targetStore, openErr = newDoltStoreFromConfig(ctx, targetBeadsDir) - } else { - targetStore, openErr = newReadOnlyStoreFromConfig(ctx, targetBeadsDir) - } - // Restore the original env var - if origDB != "" { - _ = os.Setenv("BEADS_DOLT_SERVER_DATABASE", origDB) - } else { - _ = os.Unsetenv("BEADS_DOLT_SERVER_DATABASE") - } - if openErr != nil { - return nil, fmt.Errorf("opening routed store for %s: %w", matchedRoute.Path, openErr) + targetStore, err := openRoutedTargetStore(ctx, targetBeadsDir, targetDB, writable) + if err != nil { + return nil, fmt.Errorf("opening routed store for %s: %w", matchedRoute.Path, err) } var result *RoutedResult @@ -351,6 +354,91 @@ func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable return result, nil } +// lookupPrefixRoute finds the routes.jsonl route for id and returns it with +// the target rig's .beads directory. Having no route to follow (no prefix in +// the ID, no beads directory, no routes, no matching route, or a route back to +// the current database) wraps errRoutingUnavailable; an unreadable routes.jsonl +// does not, so callers report it instead of falling through. +func lookupPrefixRoute(id string) (*prefixRoute, string, error) { + if strings.Index(id, "-") <= 0 { + return nil, "", fmt.Errorf("%w: no prefix in ID %q", errRoutingUnavailable, id) + } + + // Find the resolved beads directory (where routes.jsonl lives) + currentBeadsDir := resolveCommandBeadsDir(dbPath) + if currentBeadsDir == "" { + return nil, "", fmt.Errorf("%w: no beads directory available", errRoutingUnavailable) + } + + // Load routes from routes.jsonl + routes, err := loadPrefixRoutes(currentBeadsDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, "", fmt.Errorf("%w: no routes available", errRoutingUnavailable) + } + return nil, "", fmt.Errorf("loading routes: %w", err) + } + if len(routes) == 0 { + return nil, "", fmt.Errorf("%w: no routes available", errRoutingUnavailable) + } + + // Find the most specific route for this ID + matchedRoute := matchPrefixRoute(routes, id) + if matchedRoute == nil { + return nil, "", fmt.Errorf("%w: no route for ID %q", errRoutingUnavailable, id) + } + + // Skip if the route points to current directory (town-level, already checked) + if matchedRoute.Path == "." { + return nil, "", fmt.Errorf("%w: route points to current database", errRoutingUnavailable) + } + + // Derive the town root from the current beads dir. + // currentBeadsDir is typically /.beads + townRoot := filepath.Dir(currentBeadsDir) + + // Resolve the target rig's .beads directory + rigDir := filepath.Join(townRoot, matchedRoute.Path) + return matchedRoute, beads.FollowRedirect(filepath.Join(rigDir, ".beads")), nil +} + +// openRoutedTargetStore opens the store of a routed target, prefix or auto, +// writable or read-only. BEADS_DOLT_SERVER_DATABASE outranks the target's +// metadata.json when a store resolves its database name, and it names the +// launching workspace's database (set by the user, or by +// preserveRedirectSourceDatabase after a redirect), so while the target opens +// it is pointed at database, the target's own dolt_database. Otherwise a +// server-mode open would connect to the launcher's database on the shared +// server, and an embedded writable open would create or migrate a wrongly +// named database under the target. An empty database leaves the environment +// as it is: the target names no database of its own to prefer. +// +// A writable open never creates the target's database +// (newExistingDoltStoreFromConfig): the open runs before the lookup it serves, +// so an id that misses must leave a repository that has not been bootstrapped +// yet as it found it. A target that has been bootstrapped does run its +// open-time schema migrations on such a miss, as any bd command run in that +// repository would; they bring it to the schema this bd writes, not to +// anything the missed id asked for. +func openRoutedTargetStore(ctx context.Context, beadsDir, database string, writable bool) (storage.DoltStorage, error) { + if database != "" { + origDB := os.Getenv("BEADS_DOLT_SERVER_DATABASE") + _ = os.Setenv("BEADS_DOLT_SERVER_DATABASE", database) + defer func() { + // Restore the original env var + if origDB != "" { + _ = os.Setenv("BEADS_DOLT_SERVER_DATABASE", origDB) + } else { + _ = os.Unsetenv("BEADS_DOLT_SERVER_DATABASE") + } + }() + } + if writable { + return newExistingDoltStoreFromConfig(ctx, beadsDir) + } + return newReadOnlyStoreFromConfig(ctx, beadsDir) +} + // matchPrefixRoute returns the route whose prefix is the longest leading match // for beadID, or nil if none matches. Route prefixes end in "-", so a // multi-hyphen prefix such as "claude-os-" matches "claude-os-76l", and with @@ -396,52 +484,57 @@ func loadPrefixRoutes(beadsDir string) ([]prefixRoute, error) { return routes, scanner.Err() } -// readDoltDatabase reads the dolt_database field from a .beads/metadata.json file. -func readDoltDatabase(beadsDir string) string { +// loadDoltDatabase reads the dolt_database field from a .beads/metadata.json +// file, and reports a metadata.json it cannot read or parse; a file without the +// field returns "" and no error. +func loadDoltDatabase(beadsDir string) (string, error) { metadataPath := filepath.Join(beadsDir, "metadata.json") data, err := os.ReadFile(metadataPath) //nolint:gosec // G304: path is constructed from trusted beads directory if err != nil { - return "" + return "", err } var meta struct { DoltDatabase string `json:"dolt_database"` } - if json.Unmarshal(data, &meta) != nil { - return "" + if err := json.Unmarshal(data, &meta); err != nil { + return "", fmt.Errorf("parsing %s: %w", metadataPath, err) } - return meta.DoltDatabase + return meta.DoltDatabase, nil } -// getIssueWithRouting gets an issue by exact ID. -// Tries the local store first, then prefix-based routing, then contributor auto-routing. +// autoRouteTargetDatabase is the dolt_database an auto-routed open points its +// target at (openRoutedTargetStore). A read stays tolerant: a metadata.json +// that is unreadable or names no database yields "", and the target's own open +// decides. A write refuses both, as a prefix route does, +// because the launcher's BEADS_DOLT_SERVER_DATABASE would then pick the +// database, and an embedded writable open creates the database it is given +// when the target holds any other (newExistingDoltStoreFromConfig refuses only +// a target that holds none). // -// Returns a RoutedResult containing the issue and the store to use for related queries. -// The caller MUST call result.Close() when done to release any routed storage. -func getIssueWithRouting(ctx context.Context, localStore storage.DoltStorage, id string) (*RoutedResult, error) { - // Try local store first. - issue, err := localStore.GetIssue(ctx, id) - if err == nil { - return &RoutedResult{ - Issue: issue, - Store: localStore, - Routed: false, - ResolvedID: id, - }, nil - } - - // If not found locally, try prefix-based routing via routes.jsonl. - if isNotFoundErr(err) { - if prefixResult, prefixErr := resolveViaPrefixRouting(ctx, id); prefixErr == nil { - return prefixResult, nil +// A target with no metadata.json at all is still opened, read or written. That +// is the planning repository contributor setup creates (runContributorWizard, +// autoConfigureForkContributor) until the first routed bd create writes one +// (ensureBeadsDirForPath). Setup names its database as any workspace without +// metadata.json does, after the variable when it is set and the default +// database otherwise, so the open is pointed at the only embedded database the +// target holds, or at the default database when it holds none or several. +// Resolving the name again at use would follow the launcher's variable, which +// need not be the one setup ran with: a read would then miss the database +// setup created, and a write would create another beside it. +func autoRouteTargetDatabase(beadsDir string, writable bool) (string, error) { + database, err := loadDoltDatabase(beadsDir) + switch { + case errors.Is(err, fs.ErrNotExist): + if databases := embeddeddolt.Databases(beadsDir); len(databases) == 1 { + return databases[0], nil } + return configfile.DefaultDoltDatabase, nil + case !writable: + return database, nil + case err != nil: + return "", err + case database == "": + return "", fmt.Errorf("%s has no dolt_database field", filepath.Join(beadsDir, "metadata.json")) } - - // If not found via prefix routing, try contributor auto-routing as fallback (GH#2345). - if isNotFoundErr(err) { - if autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id, false); autoErr == nil { - return autoResult, nil - } - } - - return nil, err + return database, nil } diff --git a/cmd/bd/routed_mutation_test.go b/cmd/bd/routed_mutation_test.go new file mode 100644 index 000000000..953dac048 --- /dev/null +++ b/cmd/bd/routed_mutation_test.go @@ -0,0 +1,730 @@ +//go:build cgo + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" +) + +// setupAutoRoute builds a git repo with the given beads.role whose store +// auto-routes, through routingKey (routing.maintainer, routing.contributor, +// ...), to a planning workspace holding the issue "shared-hub" and any +// extraSeeds, and changes into the repo. It returns the repo's store and the +// repo and planning directories. +func setupAutoRoute(t *testing.T, role, routingKey string, extraSeeds ...string) (storage.DoltStorage, string, string) { + t.Helper() + initConfigForTest(t) + neutralizeRoutingModeEnv(t) + ctx := context.Background() + + tmpDir := t.TempDir() + repoDir := filepath.Join(tmpDir, "repo") + planningDir := filepath.Join(tmpDir, "planning") + + runCmd(t, tmpDir, "git", "init", repoDir) + // Force repo-local hooks so tests ignore any global hooksPath override. + runCmd(t, repoDir, "git", "config", "core.hooksPath", ".git/hooks") + runCmd(t, repoDir, "git", "config", "beads.role", role) + + primaryStore := newTestStoreIsolatedDB(t, filepath.Join(repoDir, ".beads", "beads.db"), "shared") + if err := primaryStore.SetConfig(ctx, "routing.mode", "auto"); err != nil { + t.Fatalf("set routing.mode: %v", err) + } + if err := primaryStore.SetConfig(ctx, routingKey, planningDir); err != nil { + t.Fatalf("set %s: %v", routingKey, err) + } + + planningStore := newTestStoreIsolatedDB(t, filepath.Join(planningDir, ".beads", "beads.db"), "shared") + seedIssue(t, ctx, planningStore, "shared-hub") + for _, id := range extraSeeds { + seedIssue(t, ctx, planningStore, id) + } + if err := planningStore.Close(); err != nil { + t.Fatalf("close planning store: %v", err) + } + + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + if err := os.Chdir(repoDir); err != nil { + t.Fatalf("chdir repoDir: %v", err) + } + // Role detection reads the process-wide RepoContext cache, which an + // earlier test in this binary may have built for its own, now-deleted + // repo. Rebuild it from repoDir. + resetRepoContextCachesForTest(t) + + return primaryStore, repoDir, planningDir +} + +// neutralizeRoutingModeEnv clears the environment variables that override the +// routing.mode a routing test writes into its store. Viper binds +// BEADS_ROUTING_MODE / BD_ROUTING_MODE onto the routing.mode key +// (internal/config), which outranks the store's routing.mode=auto. Hosts that +// export either var (developer shells and agent runtimes do) would otherwise +// read routing.mode=off and never route. +func neutralizeRoutingModeEnv(t *testing.T) { + t.Helper() + t.Setenv("BEADS_ROUTING_MODE", "") + t.Setenv("BD_ROUTING_MODE", "") +} + +// routedStoreIsReadOnly reports whether a routed store was opened read-only. +// The tests check this on the store itself because a server-mode read-only +// open does not refuse a write, so a write that goes through cannot tell the +// two opens apart. +func routedStoreIsReadOnly(t *testing.T, s storage.DoltStorage) bool { + t.Helper() + routedStore, ok := storage.UnwrapStore(s).(interface{ IsReadOnly() bool }) + if !ok { + t.Fatalf("routed store %T does not expose read-only state", s) + } + return routedStore.IsReadOnly() +} + +// requireWritableAutoRoute fails unless the write-intent resolution rr, made +// for what, came through an auto-route that opened its target writable. +func requireWritableAutoRoute(t *testing.T, what string, rr *RoutedResult) { + t.Helper() + if !rr.Routed { + t.Fatalf("%s resolved %s without routing; want the auto-route", what, rr.ResolvedID) + } + if routedStoreIsReadOnly(t, rr.Store) { + t.Fatalf("%s: the auto-route opened its target read-only for a write", what) + } +} + +// TestMaintainerAutoRouteMutationWritesThrough covers the widened +// openAutoRoutedStore behavior for write-intent callers beyond mol bond: +// resolveAndGetIssueForMutation is the resolver bd update uses, and a +// maintainer auto-routed store must open writable so the mutation actually +// commits in the routed database. (TestContributorAutoRouteWritesThrough +// covers the contributor route.) +func TestMaintainerAutoRouteMutationWritesThrough(t *testing.T) { + primaryStore, _, _ := setupAutoRoute(t, "maintainer", "routing.maintainer") + ctx := context.Background() + + rr, err := resolveAndGetIssueForMutation(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("resolveAndGetIssueForMutation: %v", err) + } + defer rr.Close() + requireWritableAutoRoute(t, "update", rr) + if err := rr.Store.UpdateIssue(ctx, rr.ResolvedID, map[string]interface{}{"priority": 0}, "test"); err != nil { + t.Fatalf("update through the maintainer auto-route: %v", err) + } + rr.Close() + + // Verify through a fresh routed read that the update committed in the + // planning database, not in a doomed in-memory handle. + reread, err := resolveAndGetIssueWithRouting(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("re-reading the routed issue: %v", err) + } + defer reread.Close() + if reread.Issue.Priority != 0 { + t.Fatalf("priority = %d after routed update, want 0", reread.Issue.Priority) + } +} + +// TestMaintainerAutoRouteExactMutationWritesThrough is the same pin for +// resolveAndGetIssueForMutationExact, the exact-match resolver bd comment and +// bd comments add use (#5393): the maintainer auto-route must open writable, +// as any other write-intent resolution does, and a comment written through it +// must commit in the routed database. +func TestMaintainerAutoRouteExactMutationWritesThrough(t *testing.T) { + primaryStore, _, _ := setupAutoRoute(t, "maintainer", "routing.maintainer") + ctx := context.Background() + + rr, err := resolveAndGetIssueForMutationExact(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("resolveAndGetIssueForMutationExact: %v", err) + } + defer rr.Close() + requireWritableAutoRoute(t, "exact-match comment", rr) + if _, err := addCommentDirect(ctx, rr.Store, rr.ResolvedID, "test", "routed comment"); err != nil { + t.Fatalf("comment through the maintainer auto-route: %v", err) + } + rr.Close() + + reread, err := resolveAndGetIssueWithRouting(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("re-reading the routed issue: %v", err) + } + defer reread.Close() + comments, err := reread.Store.GetIssueComments(ctx, reread.ResolvedID) + if err != nil { + t.Fatalf("reading routed comments: %v", err) + } + if len(comments) != 1 || comments[0].Text != "routed comment" { + t.Fatalf("comments = %+v after the routed comment write, want the one written", comments) + } +} + +// TestAutoRouteCloseTargetWriteIntent covers bd close's shared auto-routed +// store, which resolveCloseTargets opens with the same write intent as +// resolveAndGetIssueForMutation: a maintainer auto-route opens writable and the +// close commits in the routed database. (TestContributorAutoRouteWritesThrough +// covers the contributor route.) +func TestAutoRouteCloseTargetWriteIntent(t *testing.T) { + t.Run("maintainer_writes_through", func(t *testing.T) { + primaryStore, _, _ := setupAutoRoute(t, "maintainer", "routing.maintainer") + ctx := context.Background() + + results, cleanup, err := resolveCloseTargets(ctx, primaryStore, []string{"shared-hub"}) + if err != nil { + t.Fatalf("resolveCloseTargets: %v", err) + } + defer cleanup() + if len(results) != 1 || !results[0].Routed { + t.Fatalf("results = %+v, want one auto-routed close target", results) + } + if routedStoreIsReadOnly(t, results[0].Store) { + t.Fatalf("maintainer auto-route opened read-only for bd close") + } + if err := results[0].Store.CloseIssue(ctx, results[0].ResolvedID, "done", "test", ""); err != nil { + t.Fatalf("close through the maintainer auto-route: %v", err) + } + cleanup() + + reread, err := resolveAndGetIssueWithRouting(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("re-reading the routed issue: %v", err) + } + defer reread.Close() + if reread.Issue.Status != types.StatusClosed { + t.Fatalf("status = %s after the routed close, want %s", reread.Issue.Status, types.StatusClosed) + } + }) +} + +// TestContributorAutoRouteWritesThrough pins the contributor side of the +// write-intent resolvers: a contributor's auto-route target is their own +// planning repository, the one bd create writes to, so bd update, bd comment +// and bd close of an issue found there commit there (GH#2345, beads-0km). +// Each route must open writable, since a server-mode read-only open does not +// refuse a write (routedStoreIsReadOnly), while a read through the same route +// stays read-only. An id the route does not hold is still a plain not-found. +func TestContributorAutoRouteWritesThrough(t *testing.T) { + primaryStore, _, _ := setupAutoRoute(t, "contributor", "routing.contributor") + ctx := context.Background() + + writes := []struct { + name string + write func(id string) error + }{ + {"update (resolveAndGetIssueForMutation)", func(id string) error { + rr, err := resolveAndGetIssueForMutation(ctx, primaryStore, id) + if err != nil { + return err + } + defer rr.Close() + requireWritableAutoRoute(t, "update", rr) + return rr.Store.UpdateIssue(ctx, rr.ResolvedID, map[string]interface{}{"priority": 0}, "test") + }}, + {"comment (resolveAndGetIssueForMutationExact)", func(id string) error { + rr, err := resolveAndGetIssueForMutationExact(ctx, primaryStore, id) + if err != nil { + return err + } + defer rr.Close() + requireWritableAutoRoute(t, "comment", rr) + _, err = addCommentDirect(ctx, rr.Store, rr.ResolvedID, "test", "routed comment") + return err + }}, + {"close (resolveCloseTargets)", func(id string) error { + results, cleanup, err := resolveCloseTargets(ctx, primaryStore, []string{id}) + defer cleanup() + if err != nil { + return err + } + requireWritableAutoRoute(t, "close", results[0]) + return results[0].Store.CloseIssue(ctx, results[0].ResolvedID, "done", "test", "") + }}, + } + for _, w := range writes { + if err := w.write("shared-hub"); err != nil { + t.Fatalf("%s of a contributor-routed issue: %v", w.name, err) + } + if err := w.write("shared-zzzz"); !isNotFoundErr(err) { + t.Errorf("%s of an id the route does not hold: err = %v, want not-found", w.name, err) + } + } + + // Verify through a fresh routed read that every write committed in the + // planning database. + reread, err := resolveAndGetIssueWithRouting(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("re-reading the routed issue: %v", err) + } + defer reread.Close() + if !reread.Routed { + t.Fatalf("re-read resolved shared-hub without routing; want the contributor auto-route") + } + if !routedStoreIsReadOnly(t, reread.Store) { + t.Fatalf("the contributor auto-route opened writable for a read") + } + if reread.Issue.Priority != 0 || reread.Issue.Status != types.StatusClosed { + t.Fatalf("priority = %d, status = %s after the routed writes, want 0 and %s", reread.Issue.Priority, reread.Issue.Status, types.StatusClosed) + } + comments, err := reread.Store.GetIssueComments(ctx, reread.ResolvedID) + if err != nil { + t.Fatalf("reading routed comments: %v", err) + } + if len(comments) != 1 || comments[0].Text != "routed comment" { + t.Fatalf("comments = %+v after the routed comment write, want the one written", comments) + } +} + +// TestRoutedMutationResolversReportRoutingFailures pins that every +// write-intent resolver reports a route it cannot follow as that failure, not +// as the local "no issue found matching", which would send the user looking +// for an issue that exists. Each routing tier gets a target whose +// metadata.json is unloadable or names no dolt_database, which a write refuses +// on either tier. +func TestRoutedMutationResolversReportRoutingFailures(t *testing.T) { + t.Run("auto_route_store_fails_to_open", func(t *testing.T) { + primaryStore, _, planningDir := setupAutoRoute(t, "maintainer", "routing.maintainer") + // Corrupted after the issue is seeded, so the route resolves but its + // store cannot open. + if err := os.WriteFile(filepath.Join(planningDir, ".beads", "metadata.json"), []byte("{not json"), 0o600); err != nil { + t.Fatalf("corrupt planning metadata.json: %v", err) + } + requireMutationResolversReport(t, primaryStore, "shared-hub", "failed to open routed store") + }) + + t.Run("auto_route_target_names_no_database", func(t *testing.T) { + primaryStore, repoDir, planningDir := setupAutoRoute(t, "contributor", "routing.contributor") + // With no dolt_database of the target's own, the open would take the + // launcher's from BEADS_DOLT_SERVER_DATABASE. + launcherDB, err := loadDoltDatabase(filepath.Join(repoDir, ".beads")) + if err != nil || launcherDB == "" { + t.Fatalf("the launcher's metadata.json names no dolt_database (err = %v)", err) + } + t.Setenv("BEADS_DOLT_SERVER_DATABASE", launcherDB) + writeTestMetadata(t, filepath.Join(planningDir, ".beads", "beads.db"), "") + requireMutationResolversReport(t, primaryStore, "shared-hub", "no dolt_database field") + }) + + t.Run("prefix_route_target_unloadable", func(t *testing.T) { + initConfigForTest(t) + tmpDir := t.TempDir() + townBeadsDir := filepath.Join(tmpDir, ".beads") + rigBeadsDir := filepath.Join(tmpDir, "rig", ".beads") + for _, dir := range []string{townBeadsDir, rigBeadsDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + } + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + if err := os.WriteFile(filepath.Join(townBeadsDir, "routes.jsonl"), []byte("{\"prefix\":\"gt-\",\"path\":\"rig\"}\n"), 0o600); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + if err := os.WriteFile(filepath.Join(rigBeadsDir, "metadata.json"), []byte("{not json"), 0o600); err != nil { + t.Fatalf("write rig metadata.json: %v", err) + } + + oldDbPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDbPath }) + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + if err := os.Chdir(tmpDir); err != nil { + t.Fatalf("chdir: %v", err) + } + + requireMutationResolversReport(t, townStore, "gt-hub", "no dolt_database configured") + }) +} + +// TestRoutedWriteDoesNotCreateTargetDatabase pins that a write-intent routed +// open refuses an embedded target that holds no database yet instead of +// creating one there. The open runs before the lookup it serves, so an id that +// misses would leave an empty database behind, and `bd bootstrap` would then +// take it for the real one and skip the clone from sync.remote. Each tier +// routes to a fresh clone: an embedded metadata.json and no embeddeddolt +// directory. +func TestRoutedWriteDoesNotCreateTargetDatabase(t *testing.T) { + // Server mode from the environment would send the target's open to a + // server, which never creates a missing database; this pins the embedded + // open. + t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + + t.Run("auto_route", func(t *testing.T) { + primaryStore, _, _ := setupAutoRoute(t, "maintainer", "routing.maintainer") + targetDir := filepath.Join(t.TempDir(), "fresh-clone") + targetBeadsDir := filepath.Join(targetDir, ".beads") + writeEmbeddedMetadata(t, targetBeadsDir, "fresh") + if err := primaryStore.SetConfig(context.Background(), "routing.maintainer", targetDir); err != nil { + t.Fatalf("set routing.maintainer: %v", err) + } + + requireMutationResolversReport(t, primaryStore, "shared-zzzz", "no embedded database") + requireNoEmbeddedDatabase(t, targetBeadsDir) + }) + + t.Run("prefix_route", func(t *testing.T) { + initConfigForTest(t) + tmpDir := t.TempDir() + townBeadsDir := filepath.Join(tmpDir, ".beads") + rigBeadsDir := filepath.Join(tmpDir, "rig", ".beads") + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + if err := os.WriteFile(filepath.Join(townBeadsDir, "routes.jsonl"), []byte("{\"prefix\":\"gt-\",\"path\":\"rig\"}\n"), 0o600); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + writeEmbeddedMetadata(t, rigBeadsDir, "rig") + + oldDbPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDbPath }) + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + if err := os.Chdir(tmpDir); err != nil { + t.Fatalf("chdir: %v", err) + } + + requireMutationResolversReport(t, townStore, "gt-zzzz", "no embedded database") + requireNoEmbeddedDatabase(t, rigBeadsDir) + }) +} + +// writeEmbeddedMetadata writes the metadata.json of an embedded workspace that +// names database, as a fresh clone has before `bd bootstrap` runs. +func writeEmbeddedMetadata(t *testing.T, beadsDir, database string) { + t.Helper() + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("create %s: %v", beadsDir, err) + } + cfg := &configfile.Config{ + Database: "dolt", + Backend: configfile.BackendDolt, + DoltMode: configfile.DoltModeEmbedded, + DoltDatabase: database, + } + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("write %s metadata.json: %v", beadsDir, err) + } +} + +// requireNoEmbeddedDatabase fails if anything created beadsDir's embedded +// database directory. +func requireNoEmbeddedDatabase(t *testing.T, beadsDir string) { + t.Helper() + if _, err := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("%s: embeddeddolt stat err = %v after a routed write missed, want it absent", beadsDir, err) + } +} + +// requireMutationResolversReport resolves id through every write-intent +// resolver and fails unless each returns the routing failure containing want. +func requireMutationResolversReport(t *testing.T, localStore storage.DoltStorage, id, want string) { + t.Helper() + ctx := context.Background() + resolvers := []struct { + name string + resolve func() error + }{ + {"resolveAndGetIssueForMutation", func() error { + rr, err := resolveAndGetIssueForMutation(ctx, localStore, id) + if err == nil { + rr.Close() + } + return err + }}, + {"resolveAndGetIssueForMutationExact", func() error { + rr, err := resolveAndGetIssueForMutationExact(ctx, localStore, id) + if err == nil { + rr.Close() + } + return err + }}, + {"resolveCloseTargets", func() error { + _, cleanup, err := resolveCloseTargets(ctx, localStore, []string{id}) + cleanup() + return err + }}, + } + for _, r := range resolvers { + err := r.resolve() + if err == nil || isNotFoundErr(err) || !strings.Contains(err.Error(), want) { + t.Errorf("%s: err = %v, want the routing failure %q", r.name, err, want) + } + } +} + +// TestAutoRouteOpensTargetDatabaseDespiteEnv pins that an auto-routed store +// opens the target's own dolt_database. BEADS_DOLT_SERVER_DATABASE outranks +// metadata.json and names the launching workspace's database (set by the user, +// or by preserveRedirectSourceDatabase after a redirect), so without the +// override a routed open connects to the launcher's database, where the issue +// is not, and a writable one would commit there. +func TestAutoRouteOpensTargetDatabaseDespiteEnv(t *testing.T) { + primaryStore, repoDir, _ := setupAutoRoute(t, "maintainer", "routing.maintainer") + ctx := context.Background() + launcherDB, err := loadDoltDatabase(filepath.Join(repoDir, ".beads")) + if err != nil || launcherDB == "" { + t.Fatalf("the launcher's metadata.json names no dolt_database (err = %v)", err) + } + t.Setenv("BEADS_DOLT_SERVER_DATABASE", launcherDB) + + resolvers := []struct { + name string + resolve func(context.Context, storage.DoltStorage, string) (*RoutedResult, error) + }{ + {"resolveAndGetIssueWithRouting", resolveAndGetIssueWithRouting}, + {"resolveAndGetIssueForMutation", resolveAndGetIssueForMutation}, + } + for _, r := range resolvers { + rr, err := r.resolve(ctx, primaryStore, "shared-hub") + if err != nil { + t.Fatalf("%s with BEADS_DOLT_SERVER_DATABASE=%s: %v", r.name, launcherDB, err) + } + routed := rr.Routed + rr.Close() + if !routed { + t.Fatalf("%s resolved shared-hub without routing", r.name) + } + if got := os.Getenv("BEADS_DOLT_SERVER_DATABASE"); got != launcherDB { + t.Fatalf("%s left BEADS_DOLT_SERVER_DATABASE = %q, want %q restored", r.name, got, launcherDB) + } + } +} + +// TestAutoRouteMissWithoutMetadataIgnoresEnv pins that an auto-route to a +// target with no metadata.json opens the only database the target holds, +// whatever BEADS_DOLT_SERVER_DATABASE says at use. Contributor setup names that +// database after the variable when it is set and the default database +// otherwise, and the target's open resolves its name the same way, the +// variable first, so a launcher whose variable differed from setup's used to +// read a database that is not there and, writing, create one beside setup's: +// an embedded writable open creates the database it is given when the target +// holds another. Each target has the shape autoConfigureForkContributor +// leaves, one embedded database and no metadata.json, and a miss, read or +// written, reports not found and leaves it holding only that database. +func TestAutoRouteMissWithoutMetadataIgnoresEnv(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + primaryStore, repoDir, _ := setupAutoRoute(t, "maintainer", "routing.maintainer") + launcherDB, err := loadDoltDatabase(filepath.Join(repoDir, ".beads")) + if err != nil || launcherDB == "" || launcherDB == configfile.DefaultDoltDatabase { + t.Fatalf("the launcher's metadata.json names dolt_database %q (err = %v), want one of its own", launcherDB, err) + } + + tests := []struct { + name string + targetDB string // the database setup created + useDB string // BEADS_DOLT_SERVER_DATABASE at use + }{ + {"setup_without_variable", configfile.DefaultDoltDatabase, launcherDB}, + {"setup_with_variable", launcherDB, launcherDB}, + {"setup_with_variable_unset_at_use", launcherDB, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_DATABASE", tt.useDB) + targetDir := filepath.Join(t.TempDir(), "planning") + seedEmbeddedSchema(t, targetDir, tt.targetDB) + if err := primaryStore.SetConfig(context.Background(), "routing.maintainer", targetDir); err != nil { + t.Fatalf("set routing.maintainer: %v", err) + } + + for _, writable := range []bool{false, true} { + rr, err := resolveAndGetIssueWithRoutingAccess(context.Background(), primaryStore, "shared-zzzz", writable) + if err == nil { + rr.Close() + t.Fatalf("resolved shared-zzzz (writable %v), which no workspace holds", writable) + } + if !isNotFoundErr(err) { + t.Fatalf("resolve shared-zzzz (writable %v): err = %v, want not found", writable, err) + } + } + entries, err := os.ReadDir(filepath.Join(targetDir, ".beads", "embeddeddolt")) + if err != nil { + t.Fatalf("read the target's embeddeddolt: %v", err) + } + var databases []string + for _, entry := range entries { + if entry.IsDir() { + databases = append(databases, entry.Name()) + } + } + if len(databases) != 1 || databases[0] != tt.targetDB { + t.Fatalf("the target holds databases %v after a routed miss, want only %q", databases, tt.targetDB) + } + }) + } +} + +// TestRoutedWriteRetryCommitsFailedCommit pins that the retry of a routed write +// whose commit failed commits it. The write reaches the target's working set +// before the commit runs, so the retry finds the issue already released or +// closed and refuses; nothing else commits a routed target, so the refusal +// commits the pending write first. The run whose commit failed reports no +// success: bd unclaim neither prints the release nor emits it in --json. The +// launcher and the rig are embedded, and the commit fails on an invalid +// auto-commit mode, after the write. +func TestRoutedWriteRetryCommitsFailedCommit(t *testing.T) { + saveStorageMode(t) + saveAndRestoreGlobals(t) + serverMode, proxiedServerMode = false, false + t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + initConfigForTest(t) + ctx := context.Background() + + actorMu.Lock() + savedActor, savedPending := actor, actorGitFallbackPending + actor, actorGitFallbackPending = "alice", false + actorMu.Unlock() + savedRootCtx, savedJSON, savedDidWrite := rootCtx, jsonOutput, commandDidWrite.Load() + rootCtx = ctx + t.Cleanup(func() { + actorMu.Lock() + actor, actorGitFallbackPending = savedActor, savedPending + actorMu.Unlock() + rootCtx, jsonOutput = savedRootCtx, savedJSON + commandDidWrite.Store(savedDidWrite) + }) + + tmpDir := t.TempDir() + townBeadsDir := filepath.Join(tmpDir, ".beads") + rigDir := filepath.Join(tmpDir, "rig") + rigBeadsDir := filepath.Join(rigDir, ".beads") + + writeEmbeddedMetadata(t, rigBeadsDir, "rig") + seedEmbeddedSchema(t, rigDir, "rig") + rig, err := embeddeddolt.Open(ctx, rigBeadsDir, "rig", "main") + if err != nil { + t.Fatalf("open rig: %v", err) + } + if err := rig.SetConfig(ctx, "issue_prefix", "gt"); err != nil { + t.Fatalf("set rig issue_prefix: %v", err) + } + for _, issue := range []*types.Issue{ + {ID: "gt-rel1", Status: types.StatusInProgress, Assignee: "alice"}, + {ID: "gt-rel2", Status: types.StatusInProgress, Assignee: "alice"}, + {ID: "gt-resp", Status: types.StatusOpen}, + {ID: "gt-dism", Status: types.StatusOpen}, + } { + issue.Title, issue.Priority, issue.IssueType = "routed "+issue.ID, 2, types.TypeTask + if err := rig.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + if err := rig.Commit(ctx, "seed"); err != nil { + t.Fatalf("commit rig seed: %v", err) + } + if err := rig.Close(); err != nil { + t.Fatalf("close rig: %v", err) + } + + writeEmbeddedMetadata(t, townBeadsDir, "hq") + seedEmbeddedSchema(t, tmpDir, "hq") + town, err := embeddeddolt.Open(ctx, townBeadsDir, "hq", "main") + if err != nil { + t.Fatalf("open town: %v", err) + } + t.Cleanup(func() { _ = town.Close() }) + if err := town.SetConfig(ctx, "issue_prefix", "hq"); err != nil { + t.Fatalf("set town issue_prefix: %v", err) + } + if err := os.WriteFile(filepath.Join(townBeadsDir, "routes.jsonl"), []byte("{\"prefix\":\"gt-\",\"path\":\"rig\"}\n"), 0o600); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + store = town + storeMutex.Lock() + storeActive = true + storeMutex.Unlock() + dbPath = filepath.Join(townBeadsDir, "dolt") + t.Chdir(tmpDir) + + unclaim := func(id string) error { return unclaimCmd.RunE(unclaimCmd, []string{id}) } + released := func(issue *types.Issue) bool { return issue.Assignee == "" && issue.Status == types.StatusOpen } + closed := func(issue *types.Issue) bool { return issue.Status == types.StatusClosed } + cases := []struct { + name string + id string + json bool + run func(id string) error + written func(*types.Issue) bool + refusal string // the retry's, on the write it finds pending + }{ + {"unclaim", "gt-rel1", false, unclaim, released, "issue gt-rel1 is not assigned"}, + {"unclaim --json", "gt-rel2", true, unclaim, released, "issue gt-rel2 is not assigned"}, + {"human respond", "gt-resp", false, func(id string) error { + return humanRespondCmd.RunE(humanRespondCmd, []string{id, "approved"}) + }, closed, "issue gt-resp is already closed"}, + {"human dismiss", "gt-dism", false, func(id string) error { + return humanDismissCmd.RunE(humanDismissCmd, []string{id}) + }, closed, "issue gt-dism is already closed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + jsonOutput = tc.json + var err error + run := func() (string, string) { + return captureStdio(t, func() { err = tc.run(tc.id) }) + } + + doltAutoCommit = "bogus" + stdout, stderr := run() + if err == nil || stdout != "" { + t.Fatalf("run whose commit failed: err = %v, stdout = %q; want a failure and no output", err, stdout) + } + if !strings.Contains(stderr, `invalid --dolt-auto-commit="bogus"`) { + t.Fatalf("run whose commit failed: stderr = %q, want the commit failure", stderr) + } + if tc.written(committedIssue(t, rigBeadsDir, "rig", tc.id)) { + t.Fatalf("%s: the rig's HEAD holds the write whose commit failed", tc.id) + } + + doltAutoCommit = string(doltAutoCommitOn) + _, stderr = run() + if err == nil || !strings.Contains(stderr, tc.refusal) { + t.Fatalf("retry: err = %v, stderr = %q; want the refusal %q", err, stderr, tc.refusal) + } + if !tc.written(committedIssue(t, rigBeadsDir, "rig", tc.id)) { + t.Fatalf("%s: the retry left the write uncommitted", tc.id) + } + }) + } +} + +// committedIssue returns id as the HEAD commit of beadsDir's database holds +// it, without what the working set holds uncommitted. +func committedIssue(t *testing.T, beadsDir, database, id string) *types.Issue { + t.Helper() + s, err := embeddeddolt.Open(t.Context(), beadsDir, database, "main") + if err != nil { + t.Fatalf("open %s: %v", beadsDir, err) + } + defer func() { _ = s.Close() }() + issue, err := s.AsOf(t.Context(), id, "HEAD") + if err != nil { + t.Fatalf("read %s at HEAD: %v", id, err) + } + return issue +} diff --git a/cmd/bd/routed_test.go b/cmd/bd/routed_test.go index 828bb04d4..4830d5b1d 100644 --- a/cmd/bd/routed_test.go +++ b/cmd/bd/routed_test.go @@ -1,6 +1,10 @@ package main -import "testing" +import ( + "os" + "path/filepath" + "testing" +) // TestMatchPrefixRoute pins longest-prefix route selection: multi-hyphen // route prefixes must match their IDs instead of being shadowed by a @@ -41,3 +45,53 @@ func TestMatchPrefixRoute(t *testing.T) { }) } } + +// TestAutoRouteTargetDatabase pins the dolt_database an auto-routed open takes +// from its target's metadata.json: a write refuses a file it cannot parse or +// that names no database, which a read tolerates. A target with no +// metadata.json, the planning repository contributor setup leaves, opens the +// only embedded database it holds either way, and its default database when it +// holds none or several. +func TestAutoRouteTargetDatabase(t *testing.T) { + tests := []struct { + name string + metadata string // metadata.json content; "" writes no file + databases []string // embedded databases the target holds + want string + wantErr bool // for a writable open; a read never errs + }{ + {"names_database", `{"dolt_database":"planning"}`, []string{"proj"}, "planning", false}, + {"no_metadata_json", "", nil, "beads", false}, + {"no_metadata_json_one_database", "", []string{"proj"}, "proj", false}, + {"no_metadata_json_several_databases", "", []string{"alpha", "proj"}, "beads", false}, + {"no_database_field", `{"backend":"dolt"}`, []string{"proj"}, "", true}, + {"unparseable", "{not json", nil, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + beadsDir := t.TempDir() + for _, database := range tt.databases { + marker := filepath.Join(beadsDir, "embeddeddolt", database, ".dolt") + if err := os.MkdirAll(marker, 0o700); err != nil { + t.Fatalf("create database %s: %v", database, err) + } + if err := os.WriteFile(filepath.Join(marker, "opaque-entry"), nil, 0o600); err != nil { + t.Fatalf("fill database %s: %v", database, err) + } + } + if tt.metadata != "" { + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(tt.metadata), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + } + got, err := autoRouteTargetDatabase(beadsDir, true) + if got != tt.want || (err != nil) != tt.wantErr { + t.Errorf("writable: got (%q, %v), want (%q, error %v)", got, err, tt.want, tt.wantErr) + } + got, err = autoRouteTargetDatabase(beadsDir, false) + if got != tt.want || err != nil { + t.Errorf("read-only: got (%q, %v), want (%q, nil)", got, err, tt.want) + } + }) + } +} diff --git a/cmd/bd/routing_read.go b/cmd/bd/routing_read.go index 116874983..ad9dc1ca8 100644 --- a/cmd/bd/routing_read.go +++ b/cmd/bd/routing_read.go @@ -175,6 +175,20 @@ func printContributorRoutingNotice(ctx context.Context, localStore storage.DoltS // returned routing.RoutingRule identifies which rule matched, for callers // that print a routing notice. func openRoutedReadStore(ctx context.Context, store storage.DoltStorage) (storage.DoltStorage, bool, routing.RoutingRule, error) { + return openAutoRoutedStore(ctx, store, false) +} + +// openAutoRoutedStore opens the target selected by auto-routing. Every rule, +// maintainer, contributor and default alike, opens writable when writable is +// set, for write-intent callers (resolveAndGetIssueForMutation and its exact +// twin, bd close target resolution) that commit the change where the issue +// lives. A contributor's target is their own planning repository, the one bd +// create already writes to, so a planning issue is claimed, commented on and +// closed there (GH#2345, beads-0km). Reads stay read-only. +// +// The target opens its own dolt_database, as a prefix route does +// (openRoutedTargetStore, autoRouteTargetDatabase). +func openAutoRoutedStore(ctx context.Context, store storage.DoltStorage, writable bool) (storage.DoltStorage, bool, routing.RoutingRule, error) { repoPath, rule := determineAutoRoutedRepoPath(ctx, store) if repoPath == "" || repoPath == "." { return nil, false, routing.RuleNone, nil @@ -182,7 +196,11 @@ func openRoutedReadStore(ctx context.Context, store storage.DoltStorage) (storag targetRepoPath := routing.ExpandPath(repoPath) targetBeadsDir := filepath.Join(targetRepoPath, ".beads") - targetStore, err := newReadOnlyStoreFromConfig(ctx, targetBeadsDir) + database, err := autoRouteTargetDatabase(targetBeadsDir, writable) + if err != nil { + return nil, false, rule, fmt.Errorf("failed to open routed store at %s: target has no dolt_database configured: %w", targetRepoPath, err) + } + targetStore, err := openRoutedTargetStore(ctx, targetBeadsDir, database, writable) if err != nil { return nil, false, rule, fmt.Errorf("failed to open routed store at %s: %w", targetRepoPath, err) } diff --git a/cmd/bd/routing_role.go b/cmd/bd/routing_role.go index cbd5a3912..7f8a10989 100644 --- a/cmd/bd/routing_role.go +++ b/cmd/bd/routing_role.go @@ -143,7 +143,8 @@ func logRolePath(branch, path string, redirected bool) string { // rolePathNotARepoWarned limits warnIfRolePathIsNotARepo to one warning per // process: an ID lookup falls back to auto-routing once per missing ID -// (resolveViaAutoRouting), and every fallback re-runs role detection. +// (resolveViaAutoRoutingWithAccess), and every fallback re-runs role +// detection. var rolePathNotARepoWarned bool // warnIfRolePathIsNotARepo warns when role detection is about to read beads.role diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index da4fba918..d9a720e69 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -155,7 +155,27 @@ func acquireEmbeddedLock(beadsDir string, serverMode bool) (util.Unlocker, error // keeps that true for the dolt.shared-server layer; the one deliberate // exception is BEADS_DOLT_SHARED_SERVER, which is machine-global in every // resolver in the tree (see sharedServerModeForWorkspace). -func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.DoltStorage, err error) { +func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return openDoltStoreFromConfig(ctx, beadsDir, false) +} + +// newExistingDoltStoreFromConfig is newDoltStoreFromConfig for a write routed +// to another workspace (a prefix route or an auto-route, see +// openRoutedTargetStore): it opens that workspace's database writable, but +// refuses an embedded workspace that holds no database yet instead of creating +// one there. The routed open runs before the lookup it serves, so a created +// database would outlive an id that misses, and `bd bootstrap` would then take +// the empty database for the real one and skip the clone from sync.remote. The +// probe is embeddeddolt.HasRepository, the adapter's own marker check. A +// server-mode open already refuses a missing database: it does not set +// CreateIfMissing. +func newExistingDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return openDoltStoreFromConfig(ctx, beadsDir, true) +} + +// openDoltStoreFromConfig is the body of newDoltStoreFromConfig and +// newExistingDoltStoreFromConfig; requireExisting selects the latter. +func openDoltStoreFromConfig(ctx context.Context, beadsDir string, requireExisting bool) (s storage.DoltStorage, err error) { defer func() { s, err = activateEventsJournalStore(beadsDir, s, err) }() cfg, err := configfile.Load(beadsDir) if err != nil { @@ -179,6 +199,12 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol if effectiveServerMode(beadsDir, cfg) { return dolt.NewFromConfig(ctx, beadsDir) } + // Refuse before the sanitize migration below, which renames the legacy + // directory and rewrites metadata.json: a refused open writes nothing. + if requireExisting && !embeddeddolt.HasRepository(beadsDir) { + return nil, fmt.Errorf("embeddeddolt: no embedded database at %s; a routed write does not create one (run 'bd bootstrap' in that repository first)", + filepath.Join(beadsDir, "embeddeddolt")) + } database := configfile.DefaultDoltDatabase if cfg != nil { database = cfg.GetDoltDatabase() diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index bb683eb70..566953d21 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -89,6 +89,14 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol return nil, fmt.Errorf("%s", nocgoEmbeddedErrMsg) } +// newExistingDoltStoreFromConfig is the non-CGO twin of the CGO build's +// routed-write factory. That one differs from newDoltStoreFromConfig only by +// refusing to create a missing EMBEDDED database, and this build has no +// embedded store; a server-mode open never creates a missing database. +func newExistingDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return newDoltStoreFromConfig(ctx, beadsDir) +} + // newReadOnlyStoreFromConfig creates a read-only SQL-server-backed storage // backend. It does not activate the events journal: the store refuses writes, // so there is no mutation for a journal row to accompany (exemption is recorded diff --git a/cmd/bd/unclaim.go b/cmd/bd/unclaim.go index 3dd59a8fe..9964a36f2 100644 --- a/cmd/bd/unclaim.go +++ b/cmd/bd/unclaim.go @@ -78,8 +78,9 @@ Examples: diagHint()) } for _, id := range args { - // Resolve with prefix routing - result, err := resolveAndGetIssueWithRouting(ctx, store, id) + // Resolve with routing, opening a routed target writable so the + // unclaim commits where the issue lives (as bd update --claim does). + result, err := resolveAndGetIssueForMutation(ctx, store, id) if err != nil { fmt.Fprintf(os.Stderr, "Error resolving %s: %v\n", id, err) hasError = true @@ -97,6 +98,15 @@ Examples: if unclaimErr != nil { fmt.Fprintf(os.Stderr, "Error unclaiming %s: %v\n", fullID, unclaimErr) hasError = true + // An unclaim whose routed commit failed left its release in + // the target's working set, where this run finds the issue + // unassigned. Nothing else commits a routed target, so + // commit the release here. + if result.Issue != nil && result.Issue.Assignee == "" { + if err := commitRoutedWrite(ctx, result, "unclaim"); err != nil { + fmt.Fprintf(os.Stderr, "Error committing unclaim of %s: %v\n", fullID, err) + } + } result.Close() continue } @@ -107,6 +117,14 @@ Examples: } } + // A release whose commit failed is not reported as unclaimed. + if err := commitRoutedWrite(ctx, result, "unclaim"); err != nil { + fmt.Fprintf(os.Stderr, "Error committing unclaim of %s: %v\n", fullID, err) + hasError = true + result.Close() + continue + } + if jsonOutput { updated, _ := issueStore.GetIssue(ctx, fullID) if updated != nil { diff --git a/cmd/bd/update_embedded_test.go b/cmd/bd/update_embedded_test.go index f71511b17..bf53817f7 100644 --- a/cmd/bd/update_embedded_test.go +++ b/cmd/bd/update_embedded_test.go @@ -243,6 +243,73 @@ func TestEmbeddedUpdateRoutedStoreCommitsTargetHead(t *testing.T) { } } +// TestEmbeddedRoutedWriteMissDoesNotCreateTargetDatabase pins that a routed +// write whose id misses leaves an embedded target that has not been +// bootstrapped yet without a database. The routed open runs before the lookup +// it serves, so a database it created would outlive the miss, and `bd +// bootstrap` there would then take it for the real one and skip the clone. +// tgt-zzzz takes the prefix route and src-zzzz the maintainer auto-route; both +// lead to the same fresh clone. +func TestEmbeddedRoutedWriteMissDoesNotCreateTargetDatabase(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt update tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "src") + + targetDir := filepath.Join(dir, "fresh-clone") + if err := os.MkdirAll(targetDir, 0750); err != nil { + t.Fatal(err) + } + initGitRepoAt(t, targetDir) + targetBeadsDir := filepath.Join(targetDir, ".beads") + writeEmbeddedMetadata(t, targetBeadsDir, "tgt") + + route := `{"prefix":"tgt-","path":"fresh-clone"}` + "\n" + if err := os.WriteFile(filepath.Join(beadsDir, "routes.jsonl"), []byte(route), 0644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + roleCmd := gitCommand("config", "beads.role", "maintainer") + roleCmd.Dir = dir + if out, err := roleCmd.CombinedOutput(); err != nil { + t.Fatalf("git config beads.role: %v\n%s", err, out) + } + + env := bdEnv(dir) + runBD := func(args ...string) (string, error) { + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = env + out, err := cmd.CombinedOutput() + return string(out), err + } + for _, args := range [][]string{ + {"config", "set", "routing.mode", "auto"}, + {"config", "set", "routing.maintainer", targetDir}, + } { + if out, err := runBD(args...); err != nil { + t.Fatalf("bd %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + } + + for _, args := range [][]string{ + {"update", "tgt-zzzz", "--priority", "0"}, + {"close", "tgt-zzzz"}, + {"update", "src-zzzz", "--priority", "0"}, + {"close", "src-zzzz"}, + } { + t.Run(strings.Join(args, " "), func(t *testing.T) { + out, err := runBD(args...) + requireNoEmbeddedDatabase(t, targetBeadsDir) + if err == nil || !strings.Contains(out, "no embedded database") { + t.Fatalf("err = %v, want the routed open's refusal\n%s", err, out) + } + }) + } +} + // The TestEmbeddedUpdateFields* and TestEmbeddedUpdateLifecycle* tests were // split from TestEmbeddedUpdate (originally ~267s, measured under // --config=embedded), first into TestEmbeddedUpdateFields and diff --git a/docs/multi-agent/routing.md b/docs/multi-agent/routing.md index 426b8f1ee..894219ba4 100644 --- a/docs/multi-agent/routing.md +++ b/docs/multi-agent/routing.md @@ -45,7 +45,19 @@ flowchart TD Reads follow the same routing: with routing active, `bd list` and `bd ready` read from the routed repository, and ID lookups like `bd show` fall back to -the routed repository when a bead isn't found locally. +the routed repository when a bead isn't found locally. Commands that change an +existing bead (`bd update`, `bd close`, `bd comment`, `bd unclaim`, +`bd mol bond` and the like) follow every route, contributor, maintainer and +default alike, opening the routed repository for writing so the change commits +there. A routed write never creates the routed repository's database: if that +repository has not been bootstrapped yet, the write fails until you run +`bd bootstrap` there. A routed write also refuses a repository whose +`.beads/metadata.json` is unreadable or names no `dolt_database`, instead of +guessing which database to write. Through these routes a repository with no +`metadata.json` at all, such as a planning repository nothing has been created +in yet, opens the one embedded database it holds, or its default database when +it holds none or several; a prefix route in an orchestrator's `routes.jsonl` +always needs the target's `dolt_database`, for reads as well as writes. ## Role detection diff --git a/internal/storage/embeddeddolt/detection.go b/internal/storage/embeddeddolt/detection.go index b3adc2ead..407befe07 100644 --- a/internal/storage/embeddeddolt/detection.go +++ b/internal/storage/embeddeddolt/detection.go @@ -10,16 +10,24 @@ import ( // non-symlink directory with at least one entry, but entry names and private // repository files are never interpreted or opened. func HasRepository(beadsDir string) bool { + return len(Databases(beadsDir)) > 0 +} + +// Databases names, sorted, the databases of beadsDir's embedded Dolt +// repository: the non-symlink directories under embeddeddolt that pass +// HasRepository's .dolt marker probe. It is empty when beadsDir holds none. +func Databases(beadsDir string) []string { root := filepath.Join(beadsDir, "embeddeddolt") info, err := os.Lstat(root) if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { - return false + return nil } - databases, err := os.ReadDir(root) + entries, err := os.ReadDir(root) if err != nil { - return false + return nil } - for _, database := range databases { + var databases []string + for _, database := range entries { if !database.IsDir() || database.Type()&os.ModeSymlink != 0 { continue } @@ -27,10 +35,10 @@ func HasRepository(beadsDir string) bool { if err != nil || !marker.IsDir() || marker.Mode()&os.ModeSymlink != 0 { continue } - entries, err := os.ReadDir(filepath.Join(root, database.Name(), ".dolt")) - if err == nil && len(entries) > 0 { - return true + markerEntries, err := os.ReadDir(filepath.Join(root, database.Name(), ".dolt")) + if err == nil && len(markerEntries) > 0 { + databases = append(databases, database.Name()) } } - return false + return databases } diff --git a/internal/storage/embeddeddolt/detection_test.go b/internal/storage/embeddeddolt/detection_test.go index f26ec0062..dd012690e 100644 --- a/internal/storage/embeddeddolt/detection_test.go +++ b/internal/storage/embeddeddolt/detection_test.go @@ -3,6 +3,7 @@ package embeddeddolt import ( "os" "path/filepath" + "slices" "testing" ) @@ -75,3 +76,81 @@ func TestHasRepository(t *testing.T) { }) } } + +func TestDatabases(t *testing.T) { + // seed creates database under root with a nonempty .dolt marker. + seed := func(t *testing.T, root, database string) { + t.Helper() + marker := filepath.Join(root, database, ".dolt") + if err := os.MkdirAll(marker, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(marker, "opaque-entry"), []byte("not inspected"), 0o600); err != nil { + t.Fatal(err) + } + } + tests := []struct { + name string + setup func(t *testing.T, beadsDir string) + want []string + }{ + {name: "missing root"}, + { + name: "one database", + setup: func(t *testing.T, beadsDir string) { + seed(t, filepath.Join(beadsDir, "embeddeddolt"), "proj") + }, + want: []string{"proj"}, + }, + { + name: "several databases are sorted", + setup: func(t *testing.T, beadsDir string) { + root := filepath.Join(beadsDir, "embeddeddolt") + seed(t, root, "proj") + seed(t, root, "beads") + }, + want: []string{"beads", "proj"}, + }, + { + name: "entries failing the marker probe are skipped", + setup: func(t *testing.T, beadsDir string) { + t.Helper() + root := filepath.Join(beadsDir, "embeddeddolt") + seed(t, root, "proj") + if err := os.MkdirAll(filepath.Join(root, "empty", ".dolt"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "notes"), nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "proj"), filepath.Join(root, "linked")); err != nil { + t.Fatal(err) + } + }, + want: []string{"proj"}, + }, + { + name: "symlink root is refused", + setup: func(t *testing.T, beadsDir string) { + t.Helper() + elsewhere := t.TempDir() + seed(t, elsewhere, "proj") + if err := os.Symlink(elsewhere, filepath.Join(beadsDir, "embeddeddolt")); err != nil { + t.Fatal(err) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + beadsDir := t.TempDir() + if tt.setup != nil { + tt.setup(t, beadsDir) + } + if got := Databases(beadsDir); !slices.Equal(got, tt.want) { + t.Fatalf("Databases() = %v, want %v", got, tt.want) + } + }) + } +} From 47d5db39ffd381f07760a25f786e2b2bec102011 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 22:38:23 +0000 Subject: [PATCH 03/54] feat(http): bd heartbeat over bd serve (issues.heartbeat), and close the re-arm orphan-lease race (#7600) * feat(roles): issueops.Heartbeater role on the dolt, embedded and uow legs Adds issueops.Heartbeater.Heartbeat(HeartbeatRequest{Actor, IssueID}) -> HeartbeatResult{IssueID, Holder, HeartbeatAt, LeaseExpiresAt, Rearmed}, the lease refresh behind `bd heartbeat`, as its own role with its own accessor. - One body: internal/storage/issueops.ExecuteHeartbeatInTx validates (*HeartbeatFieldError for a blank/over-long actor or empty id), refuses a wisp, then runs RenewLeaseInTx, which is HeartbeatIssueInTx's body reporting the renewal. HeartbeatIssueInTx is now a wrapper, so the legacy HeartbeatIssue legs answer the same text and sentinels. - The refresh is fenced on the holder AND on the live claim: after the holder-fenced UPDATE, a read in the same transaction requires the issue to be in_progress under the holder (the `update leases set` prefix and the one IF() the Postgres dialect rewrites are unchanged; a Postgres port should expect more serialization retries from that read). A refusal deletes the caller's orphaned lease row, and the legs commit that refusal. - A refresh writes only the lease row: no row_lock, version, updated_at, event, journal entry, Dolt commit or hook. A re-arm (a live hand-doled claim with no lease row the caller holds) is recorded as a mutation: row_lock reminted, updated_at stamped, a lease_rearmed event, a journal entry, a version, a Dolt commit (LeaseRearmVersionCommit) and on_update. The remint makes a re-arm collide with a racing release or close instead of leaving an orphaned lease on an open or closed issue (a pre-existing race on every Dolt route). Rearmed is true only when a row was created: a same-second refresh (0 changed rows on embedded Dolt) or a holder spelled differently refreshes the existing row in place and keeps its granted_at/granted_node. Reported times are rounded to whole seconds, the stored values; the binds stay unrounded so a same-second refresh is never a 0-row UPDATE on Dolt. - Typed refusals: an absent id is ErrNotFound; a foreign holder is *LeaseConflictError{Holder} wrapping ErrAlreadyClaimed; a wisp (Ephemeral) or no live claim is *LeaseConflictError wrapping the new ErrLeaseNotHeld, which also matches ErrNotClaimable. - Legs: DoltStore and EmbeddedDoltStore through the issue-operation funnel (DOLT_ADD of the named tables, then DOLT_COMMIT -m, after the SQL commit); uow.NewHeartbeater through the new uow.RunTxScoped: a refresh or a refusal commits with the plain SQL COMMIT (no working-set scan, no Dolt commit, nothing another writer left uncommitted is swept), and a re-arm commits through the new ScopedCommitter (DOLT_ADD of exactly its tables, DOLT_COMMIT -m, never -Am). The legacy HeartbeatIssue doors commit a re-arm the same way and keep a refusal's orphan delete, and the proxied-server `bd heartbeat` route runs the role's body with the same scoped commit. - Census: storage.Storage gains Heartbeater(); HookFiringStore wraps it (fires on_update for a re-arm only, RoleFiresHooks knows it), telemetry wraps it, httpapi.timedProvider, uow.notifyingProvider (records a re-arm) and externaldeps.uowProvider build it; the journal and version guards list the heartbeat family as entry points; the httpclient unsupported shell is regenerated and Heartbeater sits on its unsupported allowlist until the wire lands. Conformance: RunHeartbeater* (18 cases) on dolt, embedded dolt and uow: refresh post-state with the stored times pinned to the result; no row_lock/current_revision/updated_at/events/journal/issue_versions/ version-control/hook change; a guarded write with the pre-heartbeat revision still lands; foreign holder refused against an aged lease; refused after close/release/reclaim/status move; wisp refused; hand-doled claim re-armed as a mutation; same-second heartbeat is not a re-arm; orphaned leases on released, closed and reassigned issues refused and deleted; a refresh and a refusal against a dirty working set make no commit and leave the other writer's change uncommitted, and a re-arm commits only its own tables; absent id; request rules; no request mutation. The http leg is waived until the wire lands. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(httpapi): POST /v0/beads/issues/{id}:heartbeat behind issues.heartbeat Publishes issueops.Heartbeater on bd serve as heartbeatIssue, the served lease refresh behind `bd heartbeat`. - Spec: the operation, HeartbeatIssueRequest {actor} and HeartbeatIssueResponse {id, holder, heartbeat_at, lease_expires_at, rearmed}; no `revision` (a refresh writes nothing to the row) and no expected_version (the fence is the holder and the live claim). issues.heartbeat joins the capability vocabulary and lease_not_held (409) the Problem.code list; `reason: ephemeral` qualifies it for a wisp. apigen and the wire-shape golden are regenerated; the golden diff is additive only, so wire_revision stays 2. - Server: route row on the custom-method dispatcher, handleHeartbeat in heartbeat.go (claim's actor rules via bodyActor, unknown member and query refusals), Config.Heartbeater as a required role (sourceRoles, roleSourceNames, the per-request uow.HeartbeaterSource accessor). failHeartbeat maps *LeaseConflictError typed fields: a foreign holder is the claim's conflict shape (already_claimed + assignee + issue_status); no live claim, a wisp included, is the new lease_not_held (+ issue_status, + reason: ephemeral for a wisp), matched before the generic arms so ErrLeaseNotHeld's ErrNotClaimable never surfaces as not_claimable. Its detail names the one state the refusing transaction found (closed, open and not claimed, an ephemeral wisp, ...). - bd serve wires the role from its peeled store (serveRoleSource gains Heartbeater); the serve stub stores, the corpus and e2e harnesses, the served harness and the encode oracle bind it. - Wire client vocabulary: OpHeartbeatIssue/issues.heartbeat and MethodHeartbeat are CLASSIFIED (the capability set-equality gate), ReasonEphemeral is pinned to the server's, and lease_not_held rebuilds *LeaseConflictError{Status, Ephemeral} wrapping ErrLeaseNotHeld in the native words (the code-table set-equality gate). No accessor yet. Tests: heartbeat_test.go (request mapping and projection, edge refusals, every role refusal with its code, members and state-specific detail, the code row, the capability). The served-http Heartbeater leg (served_heartbeater_test.go, 18 cases) runs the contract against a real bd serve through a test-local dial over the store's own dispatch, and the http-leg waivers are deleted. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(http): bd heartbeat over http through httpclient.Heartbeater The client leg of the served lease refresh, and `bd heartbeat` on the role for every route. - httpclient.Store.Heartbeater (heartbeater.go) dials POST /v0/beads/issues/{id}:heartbeat through Store.dispatch, so issues.heartbeat is preflighted: an older server is refused BEFORE the dial with *wire.CapabilityError naming the token (unwraps to *storage.ErrUnsupported{Capability: "issues.heartbeat"}); it never falls back to a re-claim or a metadata stamp. - The role's request rules are restated before the dial (validateHeartbeatRequest, pinned to storeops.ValidateHeartbeatRequest): blank/over-long actor, empty id, each a *HeartbeatFieldError. - Problem mapping: already_claimed (which the shared table rebuilds as the claim's ClaimConflictError) becomes *issueops.LeaseConflictError naming the holder and wrapping ErrAlreadyClaimed, in the native words ("issue already claimed by "); lease_not_held already arrives as LeaseConflictError (Ephemeral from `reason`); not_found is ErrNotFound; invalid_argument on `actor` is the role's field error. - Idempotent: true, the one write this client marks replayable (a repeat only moves the expiry later; a replay after a reclaim fails the fence). - Legacy Store.HeartbeatIssue rides the role. Both leave the generated shell and completeness_test's unsupported allowlist. - `bd heartbeat` takes Heartbeater() from the resolved store on the direct routes (dolt, embedded, http) and from the provider's HeartbeaterSource on the proxied route (exact id resolution in a read unit of work, then the role); runHeartbeatProxiedServer and heartbeat_proxied_server.go are deleted. Success output and exit codes are unchanged; refusal text changes per route (see CHANGELOG). Tests: heartbeater_test.go (stub server: preflight refusal naming the token with no POST, request/answer projection, each refusal's typed mapping including the wisp's reason, the 503 replay, pre-dial validation, validator parity, the legacy door). served_heartbeater_test.go runs the 18-case contract through env.subject.Heartbeater(), with HookedHeartbeater wired off a HookFiringStore. TestE2E_HeartbeatOverHTTP: real bd against an in-process serve: holder ok with revision and updated_at unchanged and a guarded write with the pre-heartbeat revision landing; a non-holder refused against an aged lease that stays untouched; closed, wisp and unclaimed refused, each naming its own state; an older server's handshake refused naming issues.heartbeat with no write. Corpus shapes gc-heartbeat, -non-holder, -closed, -wisp pending until reference_server.json advertises the token. CHANGELOG entry for the whole track, including the orphaned-lease race fix. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/38875641041 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. * fix(heartbeat): roll back a no-issue refusal; restate refresh vs re-arm Review fixes for #7600 (adopt-pr apply-fixes, iterations 1-2). Legacy HeartbeatIssue doors commit a refusal only when it read the issue. The dolt and embedded HeartbeatIssue callbacks committed every LeaseRenewalRefusal, including Found=false (the issue read failed). That refusal deletes nothing, but it can follow a fast-path UPDATE that already extended an orphan lease row's expiry, so committing it kept the extension. issueops.LeaseRenewalRefusalCommits now gates the commit on Found, the rule the Heartbeater role's HeartbeatRefusalCommits applies. New tests on both legs pin both arms: a no-issue orphan's expiry is unchanged, and a released issue's orphan is deleted. Refresh vs re-arm contract docs. Comments, Long help, the OpenAPI spec and the generated apigen still said every heartbeat writes only the lease row, makes no Dolt commit and fires no hook. A re-arm is recorded as a mutation: it remints the revision, records an event and a version entry, commits once (LeaseRearmVersionCommit) and fires on_update through the wrapping hook decorator. Restated in domain, uow, issueops, storage.go (why RoleFiresHooks answers true, so bd serve takes the heartbeater off a peeled store), cmd/bd heartbeat and serve, httpapi, httpclient, the spec (apigen regenerated), and the conformance and contract-test docs. Minor review findings: - The 409 already_claimed and lease_not_held details now end "nothing on the issue was written": the refusal may have deleted and committed the caller's orphan lease row. - docs/multi-agent/federation.md and the CHANGELOG entry describe the spelling-mismatch heartbeat: the row is refreshed in place and keeps granted_at and granted_node; only a re-arm stamps this node. The embedded spelling test pins both columns, and the lease.go comments say "refreshed in place" instead of "re-arms". - The refusal's orphan delete judges the holder under actorMatches, as the in-place refresh does (iteration 2). It matched holder = ? on the actor as sent, so a dead claim's orphan held under another spelling of the caller (gastown.mayor, heartbeat as gastown__mayor) survived the refusal until TTL. The refused closure now reads the row's holder and deletes that stored holder only when actorMatches(holder, actor). Both TestHeartbeatIssueCommitsARefusalOnlyWhenItReadTheIssue twins gain a cross-spelling arm (row deleted) and a control: another identity's row (gastown.dog-3) survives. Not changed: `bd heartbeat --json` still omits `rearmed`, since adding it changes a CLI JSON contract (follow-up). The federation.md paragraph post-dates the v1.2.2 docs pin, so documenting this main-only behaviour there is a maintainer call under engdocs/decisions/2026-07-17-docs-release-pin.md section 1. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * fix(uow): run the scoped re-arm's Dolt commit after the SQL commit The proxied-server heartbeat commits a re-arm with doltServerTx.CommitScoped, which ran DOLT_ADD per table and DOLT_COMMIT('-m') INSIDE the open transaction, before the SQL COMMIT. DOLT_ADD stages whole tables from the session's transaction root, so that Dolt commit was built from the re-arm's start snapshot: an issues row another session changed while the re-arm was open (a write still only in the working set, as in batch mode or before that writer's own Dolt commit) was staged at its old value, and HEAD disagreed with the rows every reader sees. This is the LatentLabsSpace/NEXUS#92 ordering the DoltStore funnel (runIssueOperationTxWithMessage) already avoids, and the review's major finding on this PR. CommitScoped now commits the SQL transaction first, then stages exactly the named tables (plus the versioned-history tables) and makes the Dolt commit on the same pinned connection, the uow twin of doltAddAndCommitPostTx: - a serialization failure of the SQL COMMIT is still returned, so RunTxScoped replays the unit of work; - the Dolt commit runs detached on its own budget and replays only conflicts that leave nothing behind: serialization failures and Dolt's 1105 "@autocommit transaction rolled back" (new isDoltAutocommitRollbackError, mirroring the DoltStore classifier of the same name); - nothing staged, or "nothing to commit", is not an error: a concurrent writer's Dolt commit already took the row. A "nothing to commit" past the staged check is logged, as the funnel logs it, so the missing audit line is explicable; - any other failure is logged, never returned. The data is committed, and returning it would make RunTxScoped replay a committed re-arm, which would then report a refresh. The session is poisoned rather than pooled, since its state is unknown. uow records no metrics, so that log line is the drop's only signal; bd.db.post_tx_commit_dropped counts only the funnel's. Statements go through schema.DrainCall, retiring the local drainCall fork (review nit). Tests: five sqlmock tests pin the order (SQL COMMIT before any DOLT_ADD), the swallow-and-poison path, the 1105 retry, the returned serialization failure and the empty-commit tolerance; each fails against the previous CommitScoped. TestIsDoltAutocommitRollbackError carries the DoltStore classifier's table over to the uow copy, plus the prefix boundary; dropping the prefix branch, the prefix's period, the 1105 check or the type check each fails a case. TestHeartbeaterRearmKeepsAConcurrentWrite is the real-Dolt regression: a change to another issues row made while the re-arm's transaction is open must be at HEAD after the re-arm's Dolt commit. Its working-set-only writer fails on the old ordering (dolt 2.1.8); the writer that Dolt-commits its own change passes on both and is kept as coverage. Review follow-ups folded in (comments, a file move and a test; the absorption log line above is the only behaviour change among them): - the freshRowLock invariant and its row_lock_guard_test.go mirror name the re-arm as the one deliberate remint without a lifecycle-field change, load-bearing for the release/close collision; - httpclient: "a write is never retried" (wire/client.go, stall_retry.go) and "only a read's answer is yes" (dispatch.go) now allow for the Idempotent heartbeat, and httpHeartbeater and issueops.HeartbeatResult document that a replayed heartbeat can report Rearmed false after a committed re-arm, with the guarded write's revision check as the backstop; - the proxied route moves back to cmd/bd/heartbeat_proxied_server.go, the file family cmd/bd/AGENTS.md prescribes; - the ScopedCommitter and RunTxScoped docs no longer say every other writer's uncommitted change stays out of the commit: a staged table is committed whole, so such a change to a named table rides in the re-arm's commit, as it does in the funnel, and only other tables keep theirs; - commitStagedTables says its drop is log-only, the absorption is logged, and the uow classifier gets its table test (see Tests). Not folded in: the base-owned IssueUseCase.Heartbeat / IssueSQLRepository.HeartbeatIssue door, dead in production since the proxied route moved to the Heartbeater role. Its one caller is the notifying hook-parity test (uow/notifying_test.go), and moving that case onto the Heartbeater means re-deriving its RenewLease hook semantics, so it is left for a follow-up. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- .../scripts/embedded-storage-test-shards.txt | 2 + CHANGELOG.md | 63 ++ backend/conformance/BUILD.bazel | 1 + backend/conformance/heartbeater_contract.go | 837 ++++++++++++++++++ backend/http/BUILD.bazel | 2 + backend/http/corpus/harness_test.go | 2 + backend/http/corpus/testdata/corpus.yaml | 32 + backend/http/e2e_cli_test.go | 13 +- backend/http/e2e_heartbeat_test.go | 185 ++++ backend/types.go | 1 + cmd/bd/heartbeat.go | 36 +- cmd/bd/heartbeat_proxied_integration_test.go | 23 +- cmd/bd/heartbeat_proxied_server.go | 97 +- cmd/bd/serve.go | 9 +- cmd/bd/serve_source_test.go | 18 + cmd/bd/serve_store_identity_test.go | 7 + docs/multi-agent/federation.md | 7 +- internal/httpapi/BUILD.bazel | 2 + internal/httpapi/apigen/types.gen.go | 33 +- internal/httpapi/claim.go | 7 + internal/httpapi/heartbeat.go | 178 ++++ internal/httpapi/heartbeat_test.go | 283 ++++++ internal/httpapi/problem.go | 46 + internal/httpapi/roles_test.go | 31 + internal/httpapi/routes.go | 12 + internal/httpapi/server.go | 25 +- internal/httpapi/spec/openapi.v0.yaml | 155 +++- .../httpapi/wireshape/testdata/golden.json | 47 + internal/httpclient/BUILD.bazel | 3 + internal/httpclient/accessors.go | 9 + internal/httpclient/completeness_test.go | 7 +- internal/httpclient/dispatch.go | 15 +- internal/httpclient/encode/oracle_test.go | 2 + internal/httpclient/heartbeater.go | 159 ++++ internal/httpclient/heartbeater_test.go | 372 ++++++++ internal/httpclient/served_harness_test.go | 2 + .../httpclient/served_heartbeater_test.go | 147 +++ internal/httpclient/served_test.go | 3 + internal/httpclient/unsupported_gen.go | 7 +- internal/httpclient/wire/client.go | 12 +- internal/httpclient/wire/handshake.go | 4 + internal/httpclient/wire/paths.go | 4 +- internal/httpclient/wire/paths_test.go | 1 + internal/httpclient/wire/problem.go | 30 + internal/httpclient/wire/skew_test.go | 11 +- internal/httpclient/wire/stall_retry.go | 6 +- internal/httpclient/wire/vocabulary_test.go | 4 + internal/storage/BUILD.bazel | 2 + internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/heartbeater.go | 59 ++ .../storage/dolt/heartbeater_contract_test.go | 142 +++ internal/storage/dolt/issues.go | 39 +- internal/storage/dolt/lease_test.go | 100 +++ internal/storage/domain/db/issue.go | 18 +- internal/storage/domain/issue.go | 33 +- internal/storage/embeddeddolt/BUILD.bazel | 2 + internal/storage/embeddeddolt/heartbeater.go | 60 ++ .../embeddeddolt/heartbeater_contract_test.go | 139 +++ internal/storage/embeddeddolt/issues.go | 31 +- internal/storage/embeddeddolt/lease_test.go | 110 ++- .../storage/externaldeps/uow_decorator.go | 7 +- internal/storage/hook_batch_applier_test.go | 4 + internal/storage/hook_decorator.go | 12 + internal/storage/hook_heartbeater.go | 47 + internal/storage/hook_heartbeater_test.go | 42 + internal/storage/hook_issue_operations.go | 1 + .../storage/hook_issue_operations_test.go | 3 + internal/storage/issueops/BUILD.bazel | 1 + internal/storage/issueops/aggregate.go | 13 + internal/storage/issueops/heartbeat_role.go | 135 +++ .../issueops/journal_completeness_test.go | 19 +- internal/storage/issueops/lease.go | 326 ++++++- .../storage/issueops/row_lock_guard_test.go | 5 +- .../issueops/version_completeness_test.go | 7 +- internal/storage/issueops/version_history.go | 8 + .../storage/role_accessor_decorator_test.go | 20 +- internal/storage/storage.go | 10 + internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/doltserver_tx.go | 170 +++- internal/storage/uow/doltserver_tx_test.go | 132 +++ internal/storage/uow/errors.go | 15 + internal/storage/uow/errors_test.go | 31 + internal/storage/uow/heartbeater.go | 84 ++ .../storage/uow/heartbeater_contract_test.go | 339 +++++++ internal/storage/uow/notifying.go | 39 + internal/storage/uow/notifying_parity_test.go | 4 +- internal/storage/uow/tx.go | 114 ++- internal/storage/uow/uow.go | 14 + internal/telemetry/BUILD.bazel | 1 + internal/telemetry/heartbeater.go | 36 + .../telemetry/role_accessor_decorator_test.go | 12 +- internal/types/types.go | 5 + issue_roles_external_test.go | 32 + issueops/BUILD.bazel | 2 + issueops/heartbeater.go | 182 ++++ issueops/heartbeater_test.go | 30 + 96 files changed, 5400 insertions(+), 193 deletions(-) create mode 100644 backend/conformance/heartbeater_contract.go create mode 100644 backend/http/e2e_heartbeat_test.go create mode 100644 internal/httpapi/heartbeat.go create mode 100644 internal/httpapi/heartbeat_test.go create mode 100644 internal/httpclient/heartbeater.go create mode 100644 internal/httpclient/heartbeater_test.go create mode 100644 internal/httpclient/served_heartbeater_test.go create mode 100644 internal/storage/dolt/heartbeater.go create mode 100644 internal/storage/dolt/heartbeater_contract_test.go create mode 100644 internal/storage/embeddeddolt/heartbeater.go create mode 100644 internal/storage/embeddeddolt/heartbeater_contract_test.go create mode 100644 internal/storage/hook_heartbeater.go create mode 100644 internal/storage/hook_heartbeater_test.go create mode 100644 internal/storage/issueops/heartbeat_role.go create mode 100644 internal/storage/uow/heartbeater.go create mode 100644 internal/storage/uow/heartbeater_contract_test.go create mode 100644 internal/telemetry/heartbeater.go create mode 100644 issueops/heartbeater.go create mode 100644 issueops/heartbeater_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index c8e48d224..ac7bf0dbc 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -271,6 +271,7 @@ 40 14 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType 40 14 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState 40 14 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest +40 14 TestHeartbeatIssueCommitsARefusalOnlyWhenItReadTheIssueEmbedded 40 14 TestHistory_NullTextColumns 40 15 TestCreateIssuesRejectStaleUpserts @@ -395,6 +396,7 @@ 40 25 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet 40 25 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor 40 25 TestGetDependentRecordsForIssuesEmbedded +40 25 TestHeartbeaterContract 40 25 TestIgnoredCursorHealIsIdempotent 40 25 TestIsBlockedBatchEmbedded 40 25 TestReadyWorkPageTotal diff --git a/CHANGELOG.md b/CHANGELOG.md index d598c0cda..604b85bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 capability error naming the missing token (never sent without the member); a batchApply plan is checked whole, so no partial plan is sent. A request naming no gate member is unaffected by either token. +- `bd heartbeat` works over a connected HTTP workspace. + The lease refresh is a role, `issueops.Heartbeater` + (`Heartbeat(HeartbeatRequest{Actor, IssueID}) -> HeartbeatResult{IssueID, + Holder, HeartbeatAt, LeaseExpiresAt, Rearmed}`), on every backend leg, and + `bd serve` publishes it as `POST /v0/beads/issues/{id}:heartbeat` behind the + new `issues.heartbeat` capability token (additive; no wire_revision bump), + with the new `lease_not_held` (409) problem code (`reason: ephemeral` for a + wisp). A refresh writes ONLY the lease row: the issue's `revision` is not + reminted (a guarded write against the pre-heartbeat token still lands), no + version, event, journal entry or Dolt commit is recorded, and no hook + fires. `Rearmed` is true only when the heartbeat created the caller's lease + row (a hand-doled claim's first heartbeat). A re-arm is recorded as a + mutation; a refresh is not: the re-arm remints `revision`, stamps + `updated_at`, records a `lease_rearmed` event, a journal entry and a + version, makes a Dolt commit and fires `on_update` (see Fixed); the + reported times are whole seconds, exactly what the lease row stores. The + HTTP backend's `Heartbeater()` and legacy `HeartbeatIssue` dial it; + against a server that does not advertise the token they refuse before + dialing with a typed capability error naming `issues.heartbeat`, and never + fall back to a re-claim or a metadata stamp. A + busy (503) answer is replayed, because a repeat only moves the expiry later. + `bd heartbeat` now runs the role on every route (the proxied-server route + through the provider's accessor), with the same + `{"id","status":"heartbeat","owner"}` success output and the same exit + codes. Refusal TEXT on stderr changes per route: the direct routes (Dolt + server, embedded) gain a `lease not held: ` prefix + (`heartbeat X: lease not held: issue not claimable: X status closed`); the + proxied-server route loses its + `Heartbeat: db: IssueSQLRepository.HeartbeatIssue: ` prefix and reads as the + direct routes do; over HTTP the refusal is the problem's detail, which names + the state found (`it is closed`, `an ephemeral wisp`, `open and not + claimed`) or, for a foreign holder, the native `issue already claimed by + `. An id that names no row answers not-found from the role rather + than "not claimable" (the CLI resolves the id first, so its output is + unchanged), and every refusal for a caller that holds no live claim + (closed, released, reclaimed, moved off in_progress, or a wisp) is + `issueops.ErrLeaseNotHeld`, which still matches `ErrNotClaimable`. + - `GET /v0/beads/dependencies/blocking` accepts `is_blocked`, behind the new `dependencies.blocking.isBlocked` capability token (additive; no wire_revision bump), and answers each item's stored, transitive @@ -642,6 +680,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 1,000 listed rows: +0.07 s at 1,000 rows and +0.46 s at 4,000 on embedded Dolt. A store with many wisps, whose edges the old read also scanned, is faster at every page size. +- A heartbeat can no longer resurrect a lease after a release or close. A + heartbeat re-arming a hand-doled claim (in_progress with no lease row) could + race a release or close on Dolt and leave an orphaned lease row on an open + or closed issue; every later heartbeat by the former holder then refreshed + it and answered success for a claim it no longer held. This affected every + Dolt route, including releases before this one. The re-arm is now recorded + as a mutation and remints the issue's row_lock, which every claim-ending + path rewrites, so it collides with a racing release or close and one side + is replayed against the other's result (the cost: a hand-doled claim's + first heartbeat changes its `revision` once). The refresh also requires, in + the same transaction, that the issue is still in_progress under the holder, + and a refusal deletes the caller's orphaned lease row, so a lease row + orphaned any other way never answers a heartbeat. A refresh and a refusal + still make no Dolt commit on any route; a re-arm commits only its own + tables (never `DOLT_COMMIT('-Am')`), so another writer's uncommitted + batch-mode edits to other tables stay uncommitted (a re-arm stages its + tables whole, as every other direct-route write does). On embedded Dolt, a heartbeat landing in + the same second as the previous one no longer re-grants the lease + (rewriting its `granted_at` / `granted_node`), and on every route a + heartbeat whose actor is a different spelling of the lease's holder + (`gastown.mayor` for `gastown__mayor`) refreshes the lease in place, + keeping its `granted_at` and granting replica, instead of re-granting it + under this node. For a Postgres port: the + refresh now reads the issue row, so under SERIALIZABLE expect more retried + serialization failures between a holder's own edits and its heartbeat. - **`bd gate list` works over a connected HTTP workspace** (S20). The workspace-wide listing built a raw `SearchIssues` filter, which the HTTP backend refuses, so gc's `renudge-stale-human-gates` order failed on every diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index bbb631a54..d491cd407 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -31,6 +31,7 @@ go_library( "edge_reader_contract.go", "expected_revision_contract.go", "graph_counter_contract.go", + "heartbeater_contract.go", "history_matching.go", "importer_contract.go", "issue_operations_contract.go", diff --git a/backend/conformance/heartbeater_contract.go b/backend/conformance/heartbeater_contract.go new file mode 100644 index 000000000..bde6409f0 --- /dev/null +++ b/backend/conformance/heartbeater_contract.go @@ -0,0 +1,837 @@ +package conformance + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the contract every implementation of publicops.Heartbeater +// must satisfy. Each case asserts what issueops/heartbeater.go PROMISES, cited +// by symbol. +// +// THREE LEGS, ONE BODY. dolt, embedded dolt and the unit-of-work provider all +// reach internal/storage/issueops.ExecuteHeartbeatInTx, so a per-leg failure +// here is a wrapper's failure: a commit the leg added, a hook it fired, a +// refusal that stopped matching errors.Is. +// +// THE QUIET CASE IS THE POINT. A heartbeat is the hottest write in a fleet and +// it is cheap because it writes the lease row and nothing else. Most of what +// RunHeartbeaterChangesNothingOnTheRow asserts is an ABSENCE — no revision, no +// version, no updated_at, no event, no journal entry, no history entry, no +// hook — and each absence is something a plausible wrapper would add. +// +// THE LIVE-CLAIM FENCE IS AN INVARIANT AND A PREDICATE. The RefusesAfter* +// cases hold the invariant: each ends a claim one way and asserts both that +// the lease row is gone and that the holder's next heartbeat is refused. The +// RefusesAnOrphanedLease* cases hold the predicate: a lease row that survives +// on an issue that is no longer claimed by its holder (what a re-arm racing a +// release or close can leave on Dolt) is refused AND deleted, never refreshed. +type HeartbeaterFixture struct { + // IssuePrefix namespaces the ids each assertion seeds, so several of them + // can share one database. + IssuePrefix string + // Heartbeater is the surface under test. + Heartbeater publicops.Heartbeater + // CreateIssue seeds a durable issue in the issues plane. + CreateIssue func(context.Context, *types.Issue, string) error + // CreateWisp seeds an ephemeral issue in the wisps plane. A nil CreateWisp + // SKIPS the wisp case loudly. + CreateWisp func(context.Context, *types.Issue, string) error + // Claimer takes the claim, and with it the lease row, a heartbeat + // refreshes. + Claimer publicops.Claimer + // Lifecycle ends or moves a claim (close, a status move, a hand-doled + // claim) and performs the guarded write the revision case lands. + Lifecycle publicops.Lifecycle + // Releaser gives a claim up for the release case. Nil SKIPS that case. + Releaser publicops.Releaser + // LeaseReclaimer reverts an aged claim for the reclaim case. Nil, or a nil + // Exec, SKIPS that case. + LeaseReclaimer publicops.LeaseReclaimer + // Exec runs raw statements against the backing database: the cases use it + // to age a lease. + Exec func(ctx context.Context, statements []SQLStatement) error + // ExecUncommitted runs raw statements as ANOTHER writer that leaves its + // change uncommitted in the version-controlled working set (a batch-mode + // edit): SQL-committed, no version-control commit. Nil means Exec already + // behaves so. + ExecUncommitted func(ctx context.Context, statements []SQLStatement) error + // QueryScalar runs a single-row query and scans it, so the post-state is + // read RAW rather than through the role being tested. + QueryScalar func(context.Context, string, []any, ...any) error + // CountHistory reports how many version-control entries the fixture's + // branch has. Nil SKIPS that one assertion, loudly, in the quiet case. + CountHistory func(context.Context) (int, error) + // SetJournalEnabled turns on durable journaling, so the quiet case can + // observe that a heartbeat records nothing in bd_events_journal. Nil SKIPS + // that one assertion, loudly. + SetJournalEnabled func(enabled bool) + // SetVersionedHistoryEnabled turns on dual-write issue-version history, so + // the quiet case can observe that a heartbeat records no issue_versions + // row. Nil SKIPS that one assertion, loudly. + SetVersionedHistoryEnabled func(enabled bool) + // HookedHeartbeater is the same role taken off the leg's HOOK-FIRING chain + // (storage.HookFiringStore on the store legs, uow.NewNotifyingProvider on + // the unit-of-work leg), built over the runner FiredUpdates reads. Nil, + // with FiredUpdates, SKIPS the no-hook assertion loudly. + HookedHeartbeater publicops.Heartbeater + // FiredUpdates reports, in firing order, the issue id of every on_update + // the hooked chain ran so far. + FiredUpdates func(context.Context) ([]string, error) +} + +const ( + heartbeaterHolder = "hb-holder" + heartbeaterIntruder = "hb-intruder" + heartbeaterReaper = "hb-reaper" +) + +// RunHeartbeaterRefreshesTheHoldersLease pins the refresh itself +// (Heartbeater: "pushes the lease the caller holds forward by the store's +// TTL") and every member of the HeartbeatResult, against the row READ RAW. +func RunHeartbeaterRefreshesTheHoldersLease(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + id := fixture.IssuePrefix + "-refresh" + heartbeaterSeedClaimed(t, ctx, fixture, id) + aged := time.Now().UTC().Add(-time.Minute).Truncate(time.Second) + if err := fixture.Exec(ctx, []SQLStatement{{ + Query: "UPDATE leases SET lease_expires_at = ?, heartbeat_at = ? WHERE issue_id = ?", + Args: []any{aged, aged, id}, + }}); err != nil { + t.Fatalf("age the lease on %s: %v", id, err) + } + + before := time.Now().UTC().Add(-time.Second) + result := heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + after := time.Now().UTC().Add(time.Second) + + if result.IssueID != id || result.Holder != heartbeaterHolder || result.Rearmed { + t.Fatalf("result = %+v, want {IssueID:%s Holder:%s Rearmed:false}", result, id, heartbeaterHolder) + } + if result.HeartbeatAt.Before(before) || result.HeartbeatAt.After(after) { + t.Fatalf("HeartbeatAt = %v, want the server clock at the call (between %v and %v)", result.HeartbeatAt, before, after) + } + if ttl := result.LeaseExpiresAt.Sub(result.HeartbeatAt); heartbeaterAbs(ttl-storeops.DefaultLeaseTTL) > time.Second { + t.Fatalf("LeaseExpiresAt - HeartbeatAt = %v, want the store's lease TTL %v", ttl, storeops.DefaultLeaseTTL) + } + if moved := heartbeaterScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM leases WHERE issue_id = ? AND holder = ? AND lease_expires_at > ? AND heartbeat_at > ?", + id, heartbeaterHolder, aged, aged); moved != 1 { + t.Fatal("the lease row's lease_expires_at and heartbeat_at did not move forward") + } + heartbeaterRequireStoredMatches(t, ctx, fixture, id, result, before, after) +} + +// RunHeartbeaterASameSecondHeartbeatIsNotARearm pins HeartbeatResult.Rearmed +// against the refresh that changes no value: a heartbeat landing in the same +// second as the claim or the previous heartbeat writes values identical to the +// row's (the lease columns hold whole seconds), which some engines report as +// zero rows affected. It is still a refresh of the caller's own row: Rearmed +// is false, and the lease's granted_at and granted_node do not move. +func RunHeartbeaterASameSecondHeartbeatIsNotARearm(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + for i := 0; i < 3; i++ { + id := fixture.IssuePrefix + "-samesec-" + string(rune('a'+i)) + heartbeaterSeedClaimed(t, ctx, fixture, id) + granted := heartbeaterScalar[string](t, ctx, fixture, + "SELECT CONCAT(CAST(granted_at AS CHAR), '|', COALESCE(granted_node, '')) FROM leases WHERE issue_id = ?", id) + for beat := 0; beat < 2; beat++ { + result := heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + if result.Rearmed { + t.Fatalf("heartbeat %d on %s right after the claim = %+v, want Rearmed:false: the holder's lease row existed", beat+1, id, result) + } + } + if got := heartbeaterScalar[string](t, ctx, fixture, + "SELECT CONCAT(CAST(granted_at AS CHAR), '|', COALESCE(granted_node, '')) FROM leases WHERE issue_id = ?", id); got != granted { + t.Fatalf("granted_at|granted_node on %s moved from %s to %s across refreshes; a refresh never re-grants", id, granted, got) + } + } +} + +// RunHeartbeaterChangesNothingOnTheRow pins the role's quiet promise +// (Heartbeater: "A REFRESH WRITES ONLY THE LEASE ROW"): the issue row's revision +// (row_lock), its current_revision, its updated_at, its events, the events +// journal, issue_versions, the version-control history and the hooks are all +// exactly where the claim left them. +func RunHeartbeaterChangesNothingOnTheRow(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + if fixture.SetJournalEnabled != nil { + fixture.SetJournalEnabled(true) + defer fixture.SetJournalEnabled(false) + } else { + t.Log("HeartbeaterFixture.SetJournalEnabled is nil: the journal assertion is skipped on this backend") + } + if fixture.SetVersionedHistoryEnabled != nil { + fixture.SetVersionedHistoryEnabled(true) + defer fixture.SetVersionedHistoryEnabled(false) + } else { + t.Log("HeartbeaterFixture.SetVersionedHistoryEnabled is nil: the issue_versions assertion is skipped on this backend") + } + id := fixture.IssuePrefix + "-quiet" + heartbeaterSeedClaimed(t, ctx, fixture, id) + // updated_at is stored at second precision on some backends; step past + // the claim's second so a stamp the heartbeat added cannot hide in it. + time.Sleep(1100 * time.Millisecond) + + snapshot := func() heartbeaterRowSnapshot { + return heartbeaterSnapshot(t, ctx, fixture, id) + } + before := snapshot() + fired := heartbeaterFired(t, ctx, fixture, id) + + surface := fixture.Heartbeater + if fixture.HookedHeartbeater != nil && fixture.FiredUpdates != nil { + surface = fixture.HookedHeartbeater + } else { + t.Log("HeartbeaterFixture.HookedHeartbeater/FiredUpdates is nil: the no-hook assertion is skipped on this backend") + } + heartbeat(t, ctx, surface, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + + after := snapshot() + if before != after { + t.Fatalf("a heartbeat changed the issue's durable state:\nbefore %+v\nafter %+v", before, after) + } + if got := heartbeaterFired(t, ctx, fixture, id); got != fired { + t.Fatalf("on_update fired %d time(s) for %s across a refresh, want 0: a refresh fires no hook", got-fired, id) + } +} + +// RunHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins is the +// reason the quiet case matters to a caller: the holder's revision token +// survives its own heartbeats, so a guarded write it composed from the claim +// still lands. +func RunHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.Lifecycle == nil { + t.Skip("HeartbeaterFixture.Lifecycle is nil: this backend cannot land a guarded write") + } + id := fixture.IssuePrefix + "-guarded" + revision := heartbeaterSeedClaimed(t, ctx, fixture, id) + + heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: heartbeaterHolder, IssueID: id, ExpectedVersion: &revision, + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "after two heartbeats"}}, + }); err != nil { + t.Fatalf("guarded write with the claim's revision after heartbeats = %v, want it to land", err) + } +} + +// RunHeartbeaterRefusesAForeignHolder pins the fence (Heartbeater REFUSALS: +// "an issue another actor holds: a *LeaseConflictError naming the Holder, +// wrapping ErrAlreadyClaimed"), and that the refusal writes nothing. +func RunHeartbeaterRefusesAForeignHolder(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + id := fixture.IssuePrefix + "-foreign" + heartbeaterSeedClaimed(t, ctx, fixture, id) + // Age the holder's lease first, so a heartbeat that ignored the holder + // fence would really CHANGE the row (some engines count only changed rows, + // and the columns hold whole seconds: an unaged row refreshed in the + // claim's second changes nothing and would hide the missing fence). + aged := time.Now().UTC().Add(-time.Minute).Truncate(time.Second) + if err := fixture.Exec(ctx, []SQLStatement{{ + Query: "UPDATE leases SET lease_expires_at = ?, heartbeat_at = ? WHERE issue_id = ?", + Args: []any{aged, aged, id}, + }}); err != nil { + t.Fatalf("age the lease on %s: %v", id, err) + } + lease := heartbeaterScalar[string](t, ctx, fixture, "SELECT CONCAT(CAST(lease_expires_at AS CHAR), '|', CAST(heartbeat_at AS CHAR), '|', holder) FROM leases WHERE issue_id = ?", id) + + _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: heartbeaterIntruder, IssueID: id}) + if !errors.Is(err, publicops.ErrAlreadyClaimed) { + t.Fatalf("heartbeat by a non-holder = %v, want ErrAlreadyClaimed", err) + } + if errors.Is(err, publicops.ErrLeaseNotHeld) { + t.Fatalf("heartbeat by a non-holder = %v also matches ErrLeaseNotHeld; the two refusals must stay distinct", err) + } + conflict := heartbeaterConflict(t, err) + if conflict.IssueID != id || conflict.Holder != heartbeaterHolder || conflict.Status != types.StatusInProgress { + t.Fatalf("LeaseConflictError = %+v, want {IssueID:%s Holder:%s Status:%s}", conflict, id, heartbeaterHolder, types.StatusInProgress) + } + if got := heartbeaterScalar[string](t, ctx, fixture, "SELECT CONCAT(CAST(lease_expires_at AS CHAR), '|', CAST(heartbeat_at AS CHAR), '|', holder) FROM leases WHERE issue_id = ?", id); got != lease { + t.Fatalf("a refused heartbeat moved the holder's lease from %s to %s", lease, got) + } +} + +// RunHeartbeaterCommitsOnlyItsOwnTables pins the commit discipline against a +// DIRTY working set (Heartbeater: "A REFRESH WRITES ONLY THE LEASE ROW"; "A +// RE-ARM IS RECORDED AS A MUTATION"): another writer has left an uncommitted +// versioned change (a batch-mode edit, here a config row). A refresh and a +// refusal make no version-control commit and leave that change uncommitted; +// a re-arm makes exactly one commit, of its own tables, and still leaves the +// other writer's change uncommitted rather than sweeping it in under the +// heartbeat's message. +func RunHeartbeaterCommitsOnlyItsOwnTables(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + if fixture.CountHistory == nil || fixture.Lifecycle == nil { + t.Skip("HeartbeaterFixture.CountHistory/Lifecycle is nil: this backend has no version-control history to observe") + } + refreshed := fixture.IssuePrefix + "-dirty-refresh" + heartbeaterSeedClaimed(t, ctx, fixture, refreshed) + refused := fixture.IssuePrefix + "-dirty-refused" + heartbeaterSeed(t, ctx, fixture, refused) + rearmed := fixture.IssuePrefix + "-dirty-rearm" + heartbeaterSeed(t, ctx, fixture, rearmed) + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: heartbeaterHolder, IssueID: rearmed, + Patch: publicops.IssuePatch{ + Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusInProgress}, + Assignee: publicops.Field[string]{Set: true, Value: heartbeaterHolder}, + }, + }); err != nil { + t.Fatalf("hand-dole %s: %v", rearmed, err) + } + + probe := "heartbeat.dirty-probe." + fixture.IssuePrefix + dirty := fixture.Exec + if fixture.ExecUncommitted != nil { + dirty = fixture.ExecUncommitted + } + if err := dirty(ctx, []SQLStatement{{ + Query: "REPLACE INTO config (`key`, value) VALUES (?, ?)", Args: []any{probe, "uncommitted"}, + }}); err != nil { + t.Fatalf("dirty the working set: %v", err) + } + defer func() { + _ = dirty(ctx, []SQLStatement{{Query: "DELETE FROM config WHERE `key` = ?", Args: []any{probe}}}) + }() + foreignDirty := func(when string) { + t.Helper() + if n := heartbeaterScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM dolt_status WHERE table_name = 'config'"); n == 0 { + t.Fatalf("%s: the other writer's uncommitted config change was committed; a heartbeat commits only its own tables", when) + } + } + history := func() int { + t.Helper() + n, err := fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory: %v", err) + } + return n + } + foreignDirty("before any heartbeat") + before := history() + + heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: refreshed}) + if got := history(); got != before { + t.Fatalf("a refresh moved the version-control history %d -> %d; a refresh makes no commit", before, got) + } + foreignDirty("after a refresh") + + if _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: refused}); err == nil { + t.Fatalf("heartbeat on unclaimed %s succeeded, want a refusal", refused) + } + if got := history(); got != before { + t.Fatalf("a refusal moved the version-control history %d -> %d; a refusal makes no commit", before, got) + } + foreignDirty("after a refusal") + + if result := heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: rearmed}); !result.Rearmed { + t.Fatalf("heartbeat on hand-doled %s = %+v, want Rearmed", rearmed, result) + } + if got := history(); got != before+1 { + t.Fatalf("a re-arm moved the version-control history %d -> %d, want exactly one commit", before, got) + } + foreignDirty("after a re-arm") +} + +// RunHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue and +// ...OnAClosedIssue and ...OnAnIssueSomeoneElseHolds pin the live-claim +// predicate (Heartbeater: "THE LIVE-CLAIM FENCE IS A PREDICATE"): a lease row +// that names the caller but survives on an issue that is open, closed, or +// in_progress under another assignee is refused, never refreshed, and the +// refusal deletes it (Heartbeater: the one write a refusal makes). The row +// is planted raw: it is the state a re-arm racing a release or close leaves. +func RunHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + id := fixture.IssuePrefix + "-orphan-open" + heartbeaterSeed(t, ctx, fixture, id) + heartbeaterRefusesAnOrphan(t, ctx, fixture, id, types.StatusOpen, "") +} + +func RunHeartbeaterRefusesAnOrphanedLeaseOnAClosedIssue(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + if fixture.Lifecycle == nil { + t.Skip("HeartbeaterFixture.Lifecycle is nil: this backend cannot close the claim") + } + id := fixture.IssuePrefix + "-orphan-closed" + heartbeaterSeedClaimed(t, ctx, fixture, id) + if _, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: heartbeaterHolder, IssueID: id, Reason: "done"}); err != nil { + t.Fatalf("close %s: %v", id, err) + } + heartbeaterRefusesAnOrphan(t, ctx, fixture, id, types.StatusClosed, "") +} + +func RunHeartbeaterRefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + requireHeartbeaterExec(t, fixture) + if fixture.Lifecycle == nil { + t.Skip("HeartbeaterFixture.Lifecycle is nil: this backend cannot hand-dole a claim") + } + id := fixture.IssuePrefix + "-orphan-moved" + heartbeaterSeed(t, ctx, fixture, id) + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: heartbeaterIntruder, IssueID: id, + Patch: publicops.IssuePatch{ + Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusInProgress}, + Assignee: publicops.Field[string]{Set: true, Value: heartbeaterIntruder}, + }, + }); err != nil { + t.Fatalf("hand-dole %s to %s: %v", id, heartbeaterIntruder, err) + } + heartbeaterRefusesAnOrphan(t, ctx, fixture, id, types.StatusInProgress, heartbeaterIntruder) +} + +// heartbeaterRefusesAnOrphan plants a lease row for heartbeaterHolder on id +// (which the holder does not hold), heartbeats as the holder, and asserts the +// refusal and that the row is gone. +func heartbeaterRefusesAnOrphan(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, id string, status types.Status, foreign string) { + t.Helper() + now := time.Now().UTC().Truncate(time.Second) + if err := fixture.Exec(ctx, []SQLStatement{ + {Query: "DELETE FROM leases WHERE issue_id = ?", Args: []any{id}}, + { + Query: "INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at, granted_node) VALUES (?, ?, ?, ?, ?, '')", + Args: []any{id, heartbeaterHolder, now.Add(-time.Minute), now.Add(time.Hour), now.Add(-time.Minute)}, + }, + }); err != nil { + t.Fatalf("plant an orphaned lease on %s: %v", id, err) + } + + _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + conflict := heartbeaterConflict(t, err) + if foreign != "" { + if !errors.Is(err, publicops.ErrAlreadyClaimed) || conflict.Holder != foreign { + t.Fatalf("heartbeat over an orphaned lease on an issue %s holds = %v (%+v), want ErrAlreadyClaimed naming %s", foreign, err, conflict, foreign) + } + } else if !errors.Is(err, publicops.ErrLeaseNotHeld) || conflict.Holder != "" { + t.Fatalf("heartbeat over an orphaned lease on a %s issue = %v (%+v), want ErrLeaseNotHeld", status, err, conflict) + } + if conflict.IssueID != id || conflict.Status != status || conflict.Ephemeral { + t.Fatalf("LeaseConflictError = %+v, want {IssueID:%s Status:%s Ephemeral:false}", conflict, id, status) + } + if n := heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 0 { + t.Fatalf("lease rows on %s after the refusal = %d, want 0: a refusal deletes the caller's orphaned lease", id, n) + } + // And it stays refused: the next heartbeat finds no row to refresh. + if _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}); err == nil { + t.Fatalf("a second heartbeat on %s succeeded after the orphan was refused", id) + } +} + +// heartbeaterRequireStoredMatches pins the reported times to the stored row +// (LeaseRenewal: "exactly what the lease row now holds"): lease_expires_at and +// heartbeat_at as stored are the result's, within the columns' one-second +// precision, and the stored expiry is the call's clock plus the store's TTL. +func heartbeaterRequireStoredMatches(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, id string, result publicops.HeartbeatResult, before, after time.Time) { + t.Helper() + window := func(at time.Time) (time.Time, time.Time) { return at.Add(-time.Second), at.Add(time.Second) } + hbLo, hbHi := window(result.HeartbeatAt) + exLo, exHi := window(result.LeaseExpiresAt) + if n := heartbeaterScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM leases WHERE issue_id = ? AND holder = ? AND heartbeat_at BETWEEN ? AND ? AND lease_expires_at BETWEEN ? AND ?", + id, result.Holder, hbLo, hbHi, exLo, exHi); n != 1 { + t.Fatalf("the stored lease row on %s does not hold the reported heartbeat_at %v and lease_expires_at %v (within 1s)", + id, result.HeartbeatAt, result.LeaseExpiresAt) + } + ttlLo, ttlHi := before.Add(storeops.DefaultLeaseTTL), after.Add(storeops.DefaultLeaseTTL) + if n := heartbeaterScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM leases WHERE issue_id = ? AND lease_expires_at BETWEEN ? AND ?", + id, ttlLo, ttlHi); n != 1 { + t.Fatalf("the stored lease_expires_at on %s is not the call's clock plus the lease TTL %v", id, storeops.DefaultLeaseTTL) + } + if !result.HeartbeatAt.Equal(result.HeartbeatAt.Truncate(time.Second)) || !result.LeaseExpiresAt.Equal(result.LeaseExpiresAt.Truncate(time.Second)) { + t.Fatalf("result times %v / %v carry sub-second digits the lease columns cannot hold", result.HeartbeatAt, result.LeaseExpiresAt) + } +} + +// RunHeartbeaterRefusesAfterClose, ...AfterRelease, ...AfterReclaim and +// ...AfterStatusMovedOffInProgress pin the in_progress fence through the +// lease-row invariant (Heartbeater: "a lease row exists only while its issue +// is in_progress with that holder as assignee"): each ends the claim one way, +// asserts the lease row went with it, and that the former holder's heartbeat +// is refused with ErrLeaseNotHeld, which also matches ErrNotClaimable. +func RunHeartbeaterRefusesAfterClose(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.Lifecycle == nil { + t.Skip("HeartbeaterFixture.Lifecycle is nil: this backend cannot close the claim") + } + heartbeaterRefusesAfter(t, ctx, fixture, "closed", types.StatusClosed, func(id string) error { + _, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: heartbeaterHolder, IssueID: id, Reason: "done"}) + return err + }) +} + +func RunHeartbeaterRefusesAfterRelease(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.Releaser == nil { + t.Skip("HeartbeaterFixture.Releaser is nil: this backend cannot release the claim") + } + heartbeaterRefusesAfter(t, ctx, fixture, "released", types.StatusOpen, func(id string) error { + _, err := fixture.Releaser.Release(ctx, publicops.ReleaseRequest{Actor: heartbeaterHolder, IssueID: id}) + return err + }) +} + +func RunHeartbeaterRefusesAfterReclaim(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.LeaseReclaimer == nil || fixture.Exec == nil { + t.Skip("HeartbeaterFixture.LeaseReclaimer/Exec is nil: this backend cannot reclaim the claim") + } + heartbeaterRefusesAfter(t, ctx, fixture, "reclaimed", types.StatusOpen, func(id string) error { + if err := fixture.Exec(ctx, []SQLStatement{{ + Query: "UPDATE leases SET lease_expires_at = ? WHERE issue_id = ?", + Args: []any{time.Now().UTC().Add(-time.Hour), id}, + }}); err != nil { + return err + } + result, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: heartbeaterReaper, Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if err == nil && len(result.Reclaimed) != 1 { + t.Fatalf("Reclaimed = %+v, want exactly %s", result.Reclaimed, id) + } + return err + }) +} + +func RunHeartbeaterRefusesAfterStatusMovedOffInProgress(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.Lifecycle == nil { + t.Skip("HeartbeaterFixture.Lifecycle is nil: this backend cannot move the status") + } + heartbeaterRefusesAfter(t, ctx, fixture, "blocked", types.StatusBlocked, func(id string) error { + _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: heartbeaterHolder, IssueID: id, + Patch: publicops.IssuePatch{Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusBlocked}}, + }) + return err + }) +} + +// RunHeartbeaterRefusesAWisp pins the plane rule (Heartbeater REFUSALS: "a +// wisp ... ErrLeaseNotHeld"): wisps are never leased, so even a wisp the +// caller holds in_progress is refused, and no lease row appears for it. +func RunHeartbeaterRefusesAWisp(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.CreateWisp == nil || fixture.QueryScalar == nil { + t.Skip("HeartbeaterFixture.CreateWisp/QueryScalar is nil: this backend cannot seed a wisp") + } + id := fixture.IssuePrefix + "-wisp" + if err := fixture.CreateWisp(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusInProgress, Assignee: heartbeaterHolder, + Priority: 2, IssueType: types.TypeTask, Ephemeral: true, + }, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", id, err) + } + + _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + if !errors.Is(err, publicops.ErrLeaseNotHeld) || !errors.Is(err, publicops.ErrNotClaimable) { + t.Fatalf("heartbeat on a wisp = %v, want ErrLeaseNotHeld matching ErrNotClaimable", err) + } + if !strings.Contains(err.Error(), "ephemeral") { + t.Fatalf("heartbeat on a wisp = %q, want the refusal to say the issue is ephemeral", err) + } + if conflict := heartbeaterConflict(t, err); conflict.IssueID != id || conflict.Holder != "" { + t.Fatalf("LeaseConflictError = %+v, want {IssueID:%s Holder:\"\"}", conflict, id) + } + if n := heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 0 { + t.Fatalf("lease rows for wisp %s = %d, want 0: wisps are never leased", id, n) + } +} + +// RunHeartbeaterRearmsAHandDoledClaim pins the self-heal (HeartbeatResult. +// Rearmed): an issue put in_progress with the caller as assignee by a generic +// update holds no lease row, and the holder's heartbeat grants one rather than +// refusing. The re-arm is recorded as a mutation (Heartbeater: "A RE-ARM IS +// RECORDED AS A MUTATION"): it remints the revision, which is what makes it +// collide with a racing release or close, and records a lease_rearmed event; +// the refresh after it does neither. +func RunHeartbeaterRearmsAHandDoledClaim(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + if fixture.Lifecycle == nil || fixture.QueryScalar == nil { + t.Skip("HeartbeaterFixture.Lifecycle/QueryScalar is nil: this backend cannot hand-dole a claim") + } + id := fixture.IssuePrefix + "-doled" + heartbeaterSeed(t, ctx, fixture, id) + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: heartbeaterHolder, IssueID: id, + Patch: publicops.IssuePatch{ + Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusInProgress}, + Assignee: publicops.Field[string]{Set: true, Value: heartbeaterHolder}, + }, + }); err != nil { + t.Fatalf("hand-dole %s: %v", id, err) + } + if n := heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 0 { + t.Fatalf("lease rows after a generic update = %d, want 0: the case needs a claim with no lease", n) + } + revision := heartbeaterScalar[int64](t, ctx, fixture, "SELECT row_lock FROM issues WHERE id = ?", id) + + before := time.Now().UTC().Add(-time.Second) + result := heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + after := time.Now().UTC().Add(time.Second) + if !result.Rearmed || result.Holder != heartbeaterHolder { + t.Fatalf("result = %+v, want Rearmed:true Holder:%s", result, heartbeaterHolder) + } + heartbeaterRequireStoredMatches(t, ctx, fixture, id, result, before, after) + if n := heartbeaterScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM leases WHERE issue_id = ? AND holder = ? AND lease_expires_at > ?", + id, heartbeaterHolder, time.Now().UTC()); n != 1 { + t.Fatal("the re-armed heartbeat left no live lease row for the holder") + } + rearmed := heartbeaterScalar[int64](t, ctx, fixture, "SELECT row_lock FROM issues WHERE id = ?", id) + if rearmed == revision { + t.Fatalf("row_lock stayed %d across a re-arming heartbeat; a re-arm remints it so a racing release or close collides with it", revision) + } + rearmEvents := func() int { + return heartbeaterScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = ?", id, string(types.EventLeaseRearmed)) + } + if n := rearmEvents(); n != 1 { + t.Fatalf("lease_rearmed events on %s after a re-arm = %d, want 1: a re-arm is recorded as a mutation", id, n) + } + // The next heartbeat refreshes the lease it just granted: no re-arm, and + // no revision change. + if again := heartbeat(t, ctx, fixture.Heartbeater, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}); again.Rearmed { + t.Fatalf("second heartbeat after the re-arm = %+v, want Rearmed:false", again) + } + if got := heartbeaterScalar[int64](t, ctx, fixture, "SELECT row_lock FROM issues WHERE id = ?", id); got != rearmed { + t.Fatalf("row_lock moved from %d to %d across a refresh after the re-arm", rearmed, got) + } + if n := rearmEvents(); n != 1 { + t.Fatalf("lease_rearmed events on %s after a refresh = %d, want still 1: a refresh records nothing", id, n) + } +} + +// RunHeartbeaterAnswersNotFoundForAnAbsentID pins Heartbeater REFUSALS: "an +// IssueID that names no row on either plane: ErrNotFound". +func RunHeartbeaterAnswersNotFoundForAnAbsentID(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{ + Actor: heartbeaterHolder, IssueID: fixture.IssuePrefix + "-absent", + }) + if !errors.Is(err, publicops.ErrNotFound) { + t.Fatalf("heartbeat on an absent id = %v, want ErrNotFound", err) + } +} + +// RunHeartbeaterRefusesABlankOrOverLongActor pins the request rules +// (HeartbeatRequest.Actor, Heartbeater REFUSALS): each is a +// *HeartbeatFieldError naming its field, matching ErrValidation. +func RunHeartbeaterRefusesABlankOrOverLongActor(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + id := fixture.IssuePrefix + "-refusals" + for _, test := range []struct { + name string + request publicops.HeartbeatRequest + field string + }{ + {"an empty actor", publicops.HeartbeatRequest{IssueID: id}, publicops.HeartbeatFieldActor}, + {"a blank actor", publicops.HeartbeatRequest{Actor: " \t", IssueID: id}, publicops.HeartbeatFieldActor}, + {"an over-long actor", publicops.HeartbeatRequest{Actor: strings.Repeat("a", types.MaxFieldLen+1), IssueID: id}, publicops.HeartbeatFieldActor}, + {"an empty id", publicops.HeartbeatRequest{Actor: heartbeaterHolder}, publicops.HeartbeatFieldIssueID}, + } { + t.Run(test.name, func(t *testing.T) { + _, err := fixture.Heartbeater.Heartbeat(ctx, test.request) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("Heartbeat(%s) = %v, want ErrValidation", test.name, err) + } + var field *publicops.HeartbeatFieldError + if !errors.As(err, &field) || field.Field != test.field { + t.Fatalf("Heartbeat(%s) = %v, want a *HeartbeatFieldError naming %q", test.name, err, test.field) + } + }) + } +} + +// RunHeartbeaterDoesNotMutateTheRequest pins the role's "never mutate +// caller-owned request values". +func RunHeartbeaterDoesNotMutateTheRequest(t *testing.T, ctx context.Context, fixture HeartbeaterFixture) { + t.Helper() + id := fixture.IssuePrefix + "-nomutate" + heartbeaterSeedClaimed(t, ctx, fixture, id) + request := publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id} + want := request + heartbeat(t, ctx, fixture.Heartbeater, request) + if request != want { + t.Fatalf("request after Heartbeat = %+v, want %+v", request, want) + } +} + +// heartbeaterRefusesAfter seeds and claims one issue, ends the claim with end, +// and asserts the invariant and the refusal. +func heartbeaterRefusesAfter(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, suffix string, status types.Status, end func(id string) error) { + t.Helper() + if fixture.QueryScalar == nil { + t.Skip("HeartbeaterFixture.QueryScalar is nil: this backend cannot read the lease row") + } + id := fixture.IssuePrefix + "-after-" + suffix + heartbeaterSeedClaimed(t, ctx, fixture, id) + if n := heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 1 { + t.Fatalf("lease rows for claimed %s = %d, want 1", id, n) + } + if err := end(id); err != nil { + t.Fatalf("end the claim on %s (%s): %v", id, suffix, err) + } + if n := heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 0 { + t.Fatalf("lease rows for %s %s = %d, want 0: the path that ended the claim left its lease row", suffix, id, n) + } + + _, err := fixture.Heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: heartbeaterHolder, IssueID: id}) + if !errors.Is(err, publicops.ErrLeaseNotHeld) || !errors.Is(err, publicops.ErrNotClaimable) { + t.Fatalf("heartbeat after the claim was %s = %v, want ErrLeaseNotHeld matching ErrNotClaimable", suffix, err) + } + if errors.Is(err, publicops.ErrAlreadyClaimed) { + t.Fatalf("heartbeat after the claim was %s = %v also matches ErrAlreadyClaimed", suffix, err) + } + conflict := heartbeaterConflict(t, err) + if conflict.IssueID != id || conflict.Holder != "" || conflict.Status != status { + t.Fatalf("LeaseConflictError = %+v, want {IssueID:%s Holder:\"\" Status:%s}", conflict, id, status) + } + if n := heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 0 { + t.Fatalf("a refused heartbeat left %d lease row(s) for %s", n, id) + } +} + +// heartbeaterRowSnapshot is every durable trace a heartbeat must not leave. +// A field the backend cannot observe holds -1. +type heartbeaterRowSnapshot struct { + RowLock int64 + CurrentRevision int64 + UpdatedAt string + Events int + JournalMaxSeq int64 + IssueVersions int + History int +} + +func heartbeaterSnapshot(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, id string) heartbeaterRowSnapshot { + t.Helper() + snap := heartbeaterRowSnapshot{ + RowLock: heartbeaterScalar[int64](t, ctx, fixture, "SELECT row_lock FROM issues WHERE id = ?", id), + CurrentRevision: heartbeaterScalar[int64](t, ctx, fixture, "SELECT current_revision FROM issues WHERE id = ?", id), + UpdatedAt: heartbeaterScalar[string](t, ctx, fixture, "SELECT CAST(updated_at AS CHAR) FROM issues WHERE id = ?", id), + Events: heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM events WHERE issue_id = ?", id), + JournalMaxSeq: -1, + IssueVersions: -1, + History: -1, + } + if fixture.SetJournalEnabled != nil { + snap.JournalMaxSeq = heartbeaterScalar[int64](t, ctx, fixture, "SELECT COALESCE(MAX(seq), 0) FROM bd_events_journal") + } + if fixture.SetVersionedHistoryEnabled != nil { + snap.IssueVersions = heartbeaterScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", id) + } + if fixture.CountHistory != nil { + n, err := fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory: %v", err) + } + snap.History = n + } + return snap +} + +func heartbeaterFired(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, id string) int { + t.Helper() + if fixture.HookedHeartbeater == nil || fixture.FiredUpdates == nil { + return 0 + } + ids, err := fixture.FiredUpdates(ctx) + if err != nil { + t.Fatalf("FiredUpdates: %v", err) + } + n := 0 + for _, fired := range ids { + if fired == id { + n++ + } + } + return n +} + +func requireHeartbeaterExec(t *testing.T, fixture HeartbeaterFixture) { + t.Helper() + if fixture.Exec == nil || fixture.QueryScalar == nil { + t.Skip("HeartbeaterFixture.Exec/QueryScalar is nil: this backend cannot read or age a lease row") + } +} + +func heartbeaterSeed(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, id string) { + t.Helper() + if err := fixture.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } +} + +// heartbeaterSeedClaimed creates id and claims it as heartbeaterHolder through +// the leg's Claimer, answering the claim's row version. +func heartbeaterSeedClaimed(t *testing.T, ctx context.Context, fixture HeartbeaterFixture, id string) int64 { + t.Helper() + if fixture.Claimer == nil { + t.Skip("HeartbeaterFixture.Claimer is nil: this backend cannot take the claim a heartbeat refreshes") + } + heartbeaterSeed(t, ctx, fixture, id) + claimed, err := fixture.Claimer.Claim(ctx, publicops.ClaimRequest{Actor: heartbeaterHolder, IssueID: id}) + if err != nil { + t.Fatalf("claim %s as %s: %v", id, heartbeaterHolder, err) + } + if claimed.Issue == nil { + t.Fatalf("claim %s answered no issue", id) + } + return claimed.Issue.RowVersion +} + +func heartbeat(t *testing.T, ctx context.Context, surface publicops.Heartbeater, request publicops.HeartbeatRequest) publicops.HeartbeatResult { + t.Helper() + result, err := surface.Heartbeat(ctx, request) + if err != nil { + t.Fatalf("Heartbeat(%+v): %v", request, err) + } + return result +} + +func heartbeaterConflict(t *testing.T, err error) *publicops.LeaseConflictError { + t.Helper() + var conflict *publicops.LeaseConflictError + if !errors.As(err, &conflict) { + t.Fatalf("refusal %v (%T) is not a *LeaseConflictError", err, err) + } + return conflict +} + +func heartbeaterScalar[T any](t *testing.T, ctx context.Context, fixture HeartbeaterFixture, query string, args ...any) T { + t.Helper() + var value T + if err := fixture.QueryScalar(ctx, query, args, &value); err != nil { + t.Fatalf("%s %v: %v", query, args, err) + } + return value +} + +func heartbeaterAbs(d time.Duration) time.Duration { + if d < 0 { + return -d + } + return d +} diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index 81d915655..133286b71 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -33,6 +33,7 @@ go_test( "attach_test.go", "e2e_cli_test.go", "e2e_export_test.go", + "e2e_heartbeat_test.go", "e2e_list_filters_test.go", "e2e_list_page_edges_test.go", "e2e_mol_test.go", @@ -62,6 +63,7 @@ go_test( "//internal/storage/backends", "//internal/storage/domain", "//internal/storage/embeddeddolt", + "//internal/storage/issueops", "//internal/testutil/bazeltest", ], ) diff --git a/backend/http/corpus/harness_test.go b/backend/http/corpus/harness_test.go index 135499b25..ca00eea59 100644 --- a/backend/http/corpus/harness_test.go +++ b/backend/http/corpus/harness_test.go @@ -142,6 +142,8 @@ func corpusServeConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { must("BatchGetter") cfg.LeaseReclaimer, err = s.LeaseReclaimer() must("LeaseReclaimer") + cfg.Heartbeater, err = s.Heartbeater() + must("Heartbeater") cfg.Relations, err = s.IssueRelations() must("IssueRelations") cfg.Commenter, err = s.Commenter() diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index a3535ec71..ba69ac9ee 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -96,6 +96,9 @@ files: {"key": "t-ifa", "id": "e2e-t-ifa", "title": "target ifa", "type": "task", "assignee": "worker", "metadata": {"seeded": "keep"}}, {"key": "t-death", "id": "e2e-t-death", "title": "target death", "type": "task", "assignee": "worker"}, {"key": "t-rc", "id": "e2e-t-rc", "title": "target rc", "type": "task", "assignee": "worker"}, + {"key": "t-hb", "id": "e2e-t-hb", "title": "target hb", "type": "task"}, + {"key": "t-hb2", "id": "e2e-t-hb2", "title": "target hb2", "type": "task"}, + {"key": "t-hb3", "id": "e2e-t-hb3", "title": "target hb3", "type": "task"}, {"key": "t-boot", "id": "e2e-t-boot", "title": "target boot", "type": "task"}, {"key": "t-ro", "id": "e2e-t-ro", "title": "target reopen", "type": "task"}, {"key": "t-dr", "id": "e2e-t-dr", "title": "target dep remove", "type": "task", "deps": [{"target": "t-db"}]}, @@ -710,6 +713,35 @@ shapes: compare: exit verify: [[show, --json, e2e-t-rc]] pending: {slice: "SRV-1", reason: "the reference server does not advertise issues.reclaim (B3 #12), so the client refuses at its capability preflight"} + - id: gc-heartbeat + source: gc formulas/mol-do-work.toml:47 (gc bd heartbeat -> bd heartbeat) + setup: [[update, e2e-t-hb, --claim]] + args: [heartbeat, e2e-t-hb, --json] + verify: [[show, --json, e2e-t-hb]] + pending: {slice: "reference-server-heartbeat", reason: "the reference server does not advertise issues.heartbeat, so the client refuses at its capability preflight; pinned passing by backend/http TestE2E_HeartbeatOverHTTP; remove the mark when reference_server.json is upgraded"} + - id: gc-heartbeat-non-holder + source: gc formulas/mol-do-work.toml:47 (a worker heartbeating a bead another actor holds) + setup: [[update, e2e-t-hb2, --claim, --actor, alice]] + args: [heartbeat, e2e-t-hb2] + expect_exit: 1 + expect_stderr: "(?i)claimed by|alice" + compare: exit + pending: {slice: "reference-server-heartbeat", reason: "the reference server does not advertise issues.heartbeat, so the client refuses at its capability preflight; pinned passing by backend/http TestE2E_HeartbeatOverHTTP; remove the mark when reference_server.json is upgraded"} + - id: gc-heartbeat-closed + source: gc formulas/mol-do-work.toml:47 (a worker heartbeating a bead that was closed under it) + setup: [[update, e2e-t-hb3, --claim], [close, e2e-t-hb3]] + args: [heartbeat, e2e-t-hb3] + expect_exit: 1 + expect_stderr: "(?i)status closed|is closed" + compare: exit + pending: {slice: "reference-server-heartbeat", reason: "the reference server does not advertise issues.heartbeat, so the client refuses at its capability preflight; pinned passing by backend/http TestE2E_HeartbeatOverHTTP; remove the mark when reference_server.json is upgraded"} + - id: gc-heartbeat-wisp + source: gc formulas/mol-do-work.toml:47 (a heartbeat on a wisp, which is never leased) + args: [heartbeat, e2e-wisp-2] + expect_exit: 1 + expect_stderr: "(?i)ephemeral" + compare: exit + pending: {slice: "reference-server-heartbeat", reason: "the reference server does not advertise issues.heartbeat, so the client refuses at its capability preflight; pinned passing by backend/http TestE2E_HeartbeatOverHTTP; remove the mark when reference_server.json is upgraded"} - id: mc-promote source: mc mol-normalized-issue-pr-v1 (promote --reason) args: [promote, e2e-wisp-p, --reason, "keep it"] diff --git a/backend/http/e2e_cli_test.go b/backend/http/e2e_cli_test.go index 096926365..47a07e3d6 100644 --- a/backend/http/e2e_cli_test.go +++ b/backend/http/e2e_cli_test.go @@ -733,6 +733,15 @@ func startE2EServer(t *testing.T) string { // the one a prior `bd connect` already pinned, to prove the pin is honored // rather than silently replaced. func startE2EServerWithIdentity(t *testing.T, projectID, database string) string { + t.Helper() + addr, _ := startE2EServerWithStore(t, projectID, database) + return addr +} + +// startE2EServerWithStore is startE2EServerWithIdentity that also hands back +// the reference store the server serves, for a test that must plant or age +// state no wire verb can reach (an aged lease row). +func startE2EServerWithStore(t *testing.T, projectID, database string) (string, *embeddeddolt.EmbeddedDoltStore) { t.Helper() beadsDir := t.TempDir() ctx := context.Background() @@ -761,7 +770,7 @@ func startE2EServerWithIdentity(t *testing.T, projectID, database string) string t.Error("the in-process server did not shut down") } }) - return srv.Addr() + return srv.Addr(), reference } func e2eServeConfig(t *testing.T, s storage.DoltStorage, projectID, database string) httpapi.Config { @@ -803,6 +812,8 @@ func e2eServeConfig(t *testing.T, s storage.DoltStorage, projectID, database str fail("BatchGetter") cfg.LeaseReclaimer, err = s.LeaseReclaimer() fail("LeaseReclaimer") + cfg.Heartbeater, err = s.Heartbeater() + fail("Heartbeater") cfg.Relations, err = s.IssueRelations() fail("IssueRelations") cfg.Commenter, err = s.Commenter() diff --git a/backend/http/e2e_heartbeat_test.go b/backend/http/e2e_heartbeat_test.go new file mode 100644 index 000000000..6d67619dd --- /dev/null +++ b/backend/http/e2e_heartbeat_test.go @@ -0,0 +1,185 @@ +//go:build cgo + +package bdhttp_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// TestE2E_HeartbeatOverHTTP runs `bd heartbeat ` — the keepalive gc's +// mol-do-work formula issues every ~90s (`gc bd heartbeat`, corpus shapes +// gc-heartbeat*) — through the real `bd` binary against an in-process `bd +// serve`. +// +// The CLI takes the store's Heartbeater, which over HTTP dials POST +// /v0/beads/issues/{id}:heartbeat. What a worker relies on is pinned here end +// to end: exit 0 and the {"id","status":"heartbeat","owner"} shape for the +// holder, with the row's revision and updated_at UNCHANGED (the heartbeat +// writes only the lease, so the holder's guarded writes keep their token); +// exit 1 for a foreign holder (whose lease stays untouched), a closed issue, a +// wisp and an unclaimed issue, each saying which. Against a server that does +// not advertise issues.heartbeat (an older server's handshake shape) the +// client refuses BEFORE the dial, naming the token, and never falls back. +func TestE2E_HeartbeatOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr, reference := startE2EServerWithStore(t, e2eProjectID, e2eDatabase) + leaseRow := func(id string) string { + t.Helper() + var row string + if err := reference.RunEventsMaintenanceTx(context.Background(), func(ctx context.Context, tx issueops.DBTX) error { + return tx.QueryRowContext(ctx, + "SELECT CONCAT(holder, '|', CAST(lease_expires_at AS CHAR), '|', CAST(heartbeat_at AS CHAR)) FROM leases WHERE issue_id = ?", id, + ).Scan(&row) + }); err != nil { + t.Fatalf("read the lease row on %s: %v", id, err) + } + return row + } + ageLease := func(id string) { + t.Helper() + aged := time.Now().UTC().Add(-time.Minute).Truncate(time.Second) + if err := reference.RunEventsMaintenanceTx(context.Background(), func(ctx context.Context, tx issueops.DBTX) error { + _, err := tx.ExecContext(ctx, "UPDATE leases SET lease_expires_at = ?, heartbeat_at = ? WHERE issue_id = ?", aged, aged, id) + return err + }); err != nil { + t.Fatalf("age the lease on %s: %v", id, err) + } + } + + heartbeatPosts := func(paths []string) int { + n := 0 + for _, p := range paths { + if strings.HasPrefix(p, "POST ") && strings.HasSuffix(p, ":heartbeat") { + n++ + } + } + return n + } + + t.Run("advertised", func(t *testing.T) { + proxyAddr, counter := startCountingProxy(t, addr) + f := newHTTPFixture(t, bin, proxyAddr) + + id := f.create("heartbeat target") + f.mustRun(nil, "update", id, "--claim", "--actor", "worker-7") + before := f.mustRun(nil, "show", id, "--json").stdout + revision := firstJSONField(t, before, "revision") + if revision == "" { + t.Fatalf("bd show --json reported no revision: %s", before) + } + updatedAt := firstJSONField(t, before, "updated_at") + + counter.reset() + r := f.mustRun(nil, "heartbeat", id, "--actor", "worker-7", "--json") + var out map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(r.stdout)), &out); err != nil { + t.Fatalf("parse bd heartbeat --json: %v\n%s", err, r.stdout) + } + if out["id"] != id || out["status"] != "heartbeat" || out["owner"] != "worker-7" { + t.Errorf("bd heartbeat --json = %v, want {id: %s, status: heartbeat, owner: worker-7}", out, id) + } + if n := heartbeatPosts(counter.snapshot()); n != 1 { + t.Errorf("bd heartbeat sent %d :heartbeat requests, want exactly one: %v", n, counter.snapshot()) + } + for _, p := range counter.snapshot() { + if strings.HasPrefix(p, "PATCH ") || (strings.HasPrefix(p, "POST ") && !strings.HasSuffix(p, ":heartbeat")) { + t.Errorf("bd heartbeat sent %s; a heartbeat writes nothing but the lease", p) + } + } + + after := f.mustRun(nil, "show", id, "--json").stdout + if got := firstJSONField(t, after, "revision"); got != revision { + t.Errorf("revision moved across a heartbeat: %s -> %s; a guarded write against the pre-heartbeat token would now fail", revision, got) + } + if got := firstJSONField(t, after, "updated_at"); got != updatedAt { + t.Errorf("updated_at moved across a heartbeat: %s -> %s", updatedAt, got) + } + // The holder's guarded write with the pre-heartbeat token still lands. + f.mustRun(nil, "update", id, "--notes", "still mine", "--if-revision", revision, "--actor", "worker-7") + + // A second heartbeat is fine: it only moves the expiry later. + f.mustRun(nil, "heartbeat", id, "--actor", "worker-7") + }) + + t.Run("refusals", func(t *testing.T) { + f := newHTTPFixture(t, bin, addr) + + foreign := f.create("held by alice") + f.mustRun(nil, "update", foreign, "--claim", "--actor", "alice") + // Aged first, so a heartbeat that ignored the holder fence would + // really change the row rather than rewrite the claim's own second. + ageLease(foreign) + held := leaseRow(foreign) + r := f.run(nil, "heartbeat", foreign, "--actor", "bob") + if r.code != 1 || !strings.Contains(r.stderr, "claimed by alice") { + t.Errorf("heartbeat by a non-holder: exit %d stderr=%s, want exit 1 naming the holder", r.code, r.stderr) + } + if got := leaseRow(foreign); got != held { + t.Errorf("a refused heartbeat by a non-holder moved alice's lease: %s -> %s", held, got) + } + + // Each lease_not_held names the one state it found, and no other. + says := func(what, stderr, want string, not ...string) { + t.Helper() + if !strings.Contains(stderr, want) { + t.Errorf("heartbeat on %s: stderr=%s, want it to say %q", what, stderr, want) + } + for _, n := range not { + if strings.Contains(stderr, n) { + t.Errorf("heartbeat on %s: stderr=%s says %q, which is not the state it found", what, stderr, n) + } + } + } + + closed := f.create("closed under its worker") + f.mustRun(nil, "update", closed, "--claim", "--actor", "worker-7") + f.mustRun(nil, "close", closed, "--actor", "worker-7") + r = f.run(nil, "heartbeat", closed, "--actor", "worker-7") + if r.code != 1 { + t.Errorf("heartbeat on a closed issue exited %d, want 1: %s", r.code, r.stderr) + } + says("a closed issue", r.stderr, "is closed", "ephemeral", "not claimed") + + wisp := f.create("a wisp is never leased", "--ephemeral") + r = f.run(nil, "heartbeat", wisp, "--actor", "worker-7") + if r.code != 1 { + t.Errorf("heartbeat on a wisp exited %d, want 1: %s", r.code, r.stderr) + } + says("a wisp", r.stderr, "ephemeral wisp", "closed", "not claimed") + + unclaimed := f.create("never claimed") + r = f.run(nil, "heartbeat", unclaimed, "--actor", "worker-7") + if r.code != 1 { + t.Errorf("heartbeat on an unclaimed issue exited %d, want 1: %s", r.code, r.stderr) + } + says("an unclaimed issue", r.stderr, "not claimed", "closed", "ephemeral") + }) + + t.Run("older server mask", func(t *testing.T) { + proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) + f := newHTTPFixture(t, bin, proxyAddr) + id := f.create("an old server cannot refresh this lease") + f.mustRun(nil, "update", id, "--claim", "--actor", "worker-7") + + counter.reset() + r := f.run(nil, "heartbeat", id, "--actor", "worker-7") + if r.code != 1 { + t.Fatalf("bd heartbeat against a server without issues.heartbeat exited %d, want 1: %s", r.code, r.stdout) + } + if !strings.Contains(r.stderr, "issues.heartbeat") { + t.Errorf("refusal does not name the missing capability: %s", r.stderr) + } + for _, p := range counter.snapshot() { + if !strings.HasPrefix(p, "GET ") { + t.Errorf("a refused heartbeat sent %s; the refusal must come before any write", p) + } + } + }) +} diff --git a/backend/types.go b/backend/types.go index bf3fde0ab..6b896fa4c 100644 --- a/backend/types.go +++ b/backend/types.go @@ -118,6 +118,7 @@ const ( EventLabelAdded = types.EventLabelAdded EventLabelRemoved = types.EventLabelRemoved EventLeaseReclaimed = types.EventLeaseReclaimed + EventLeaseRearmed = types.EventLeaseRearmed EventCompacted = types.EventCompacted ) diff --git a/cmd/bd/heartbeat.go b/cmd/bd/heartbeat.go index 6790047a7..28af08444 100644 --- a/cmd/bd/heartbeat.go +++ b/cmd/bd/heartbeat.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) var heartbeatCmd = &cobra.Command{ @@ -24,10 +25,13 @@ it up. Heartbeat pushes lease_expires_at forward and stamps heartbeat_at = now. Only the current owner may heartbeat. If the lease has already been reclaimed or the issue closed, heartbeat fails so the worker learns to stop. -Leases live in an ephemeral, node-local table: heartbeats write no Dolt commit -and no history, so any cadence comfortably below the TTL is fine. Leases are -only enforceable on the node that granted them; cross-machine claim visibility -rides the issue's status and assignee, which do commit. +Leases live in an ephemeral, node-local table: a heartbeat that refreshes your +lease writes no Dolt commit and no history, so any cadence comfortably below +the TTL is fine. Only a re-arm (the first heartbeat of a claim that has no +lease, such as one set by a plain update) is recorded as a change to the issue +and committed, once. Leases are only enforceable on the node that granted them; +cross-machine claim visibility rides the issue's status and assignee, which do +commit. Examples: bd heartbeat bd-123 @@ -48,8 +52,15 @@ Examples: ctx := rootCtx id := args[0] + // One role on every route: the store's own + // Heartbeater accessor on the direct routes — dolt, embedded dolt and + // a registered HTTP backend, where it dials issues.heartbeat and an + // older server is refused before the dial naming the token — and the + // proxied-server provider's on the proxied one. On every leg a refresh + // writes ONLY the lease row (no revision remint, no history, no Dolt + // commit); a re-arm is recorded as a mutation and committed once. if usesProxiedServer() { - return runHeartbeatProxiedServer(ctx, id) + return runHeartbeatProxied(ctx, id) } result, err := resolveAndGetIssueForMutation(ctx, store, id) @@ -68,7 +79,14 @@ Examples: defer result.Close() issueStore := result.Store - if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, currentActor()); err != nil { + heartbeater, err := issueStore.Heartbeater() + if err != nil { + return HandleErrorRespectJSON("heartbeat %s: %v", result.ResolvedID, err) + } + if _, err := heartbeater.Heartbeat(ctx, issueops.HeartbeatRequest{ + Actor: currentActor(), + IssueID: result.ResolvedID, + }); err != nil { return HandleErrorRespectJSON("heartbeat %s: %v", result.ResolvedID, err) } @@ -85,9 +103,9 @@ Examples: }, } -// renderHeartbeatSuccess prints the success shape both storage routes share — -// the classic path and the proxied-server path (heartbeat_proxied_server.go) -// call exactly this code, so the --json contract workers parse +// renderHeartbeatSuccess prints the success shape every storage route shares — +// the direct path and the proxied-server path (runHeartbeatProxied) call +// exactly this code, so the --json contract workers parse // ({"id","status":"heartbeat","owner"}) cannot drift between modes. func renderHeartbeatSuccess(id, title string) error { if jsonOutput { diff --git a/cmd/bd/heartbeat_proxied_integration_test.go b/cmd/bd/heartbeat_proxied_integration_test.go index 9c06a736f..f6591fa90 100644 --- a/cmd/bd/heartbeat_proxied_integration_test.go +++ b/cmd/bd/heartbeat_proxied_integration_test.go @@ -16,10 +16,11 @@ import ( // plane (bd-aq0ql). The worker contract is the exit code (workers call // heartbeat every ~90s and only check rc), the --json success shape is // {"id","status":"heartbeat","owner"}, and — the bd-lrgn1 invariant — a -// heartbeat mints exactly ZERO Dolt commits: the lease write lands in the +// refresh mints exactly ZERO Dolt commits: the lease write lands in the // dolt_ignored leases table via the SQL-only ephemeral commit, with the same // timestamps/format classic `bd reclaim` staleness semantics consume (the -// SQL body is literally issueops.HeartbeatIssueInTx in both modes). +// SQL body is literally issueops.HeartbeatIssueInTx in both modes). A re-arm +// is recorded as a mutation and mints exactly one (rearms_hand_doled_claim). func TestProxiedServerHeartbeat(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() @@ -209,9 +210,9 @@ func TestProxiedServerHeartbeat(t *testing.T) { t.Run("rearms_hand_doled_claim", func(t *testing.T) { // A claim hand-doled through a generic update never arms a lease // (bd-9hpgf); the holder's first heartbeat opts into lease semantics. - // This is the INSERT path of the lease write, so it doubles as the - // zero-Dolt-commit proof for UpsertLeaseInTx under the ephemeral - // commit. + // That re-arm is recorded as a mutation (row_lock reminted so it + // collides with a racing release or close), so it mints exactly one + // Dolt commit; the refresh after it mints none. issue := bdProxiedCreate(t, bd, p.dir, "Hand-doled claim", "--type", "task") bdProxiedUpdate(t, bd, p.dir, issue.ID, "--assignee", "hand-worker", "--status", "in_progress") if _, _, _, ok := leaseRow(t, issue.ID); ok { @@ -223,8 +224,9 @@ func TestProxiedServerHeartbeat(t *testing.T) { if err != nil { t.Fatalf("re-arm heartbeat: %v\nstdout:\n%s\nstderr:\n%s", err, out, stderr) } - if commitsAfter := doltCommits(t); commitsAfter != commitsBefore { - t.Errorf("re-arm heartbeat minted Dolt commits: %d -> %d", commitsBefore, commitsAfter) + commitsAfter := doltCommits(t) + if commitsAfter != commitsBefore+1 { + t.Errorf("re-arm heartbeat Dolt commits: %d -> %d, want exactly one (a re-arm is recorded as a mutation)", commitsBefore, commitsAfter) } holder, expires, _, ok := leaseRow(t, issue.ID) if !ok || holder != "hand-worker" { @@ -233,5 +235,12 @@ func TestProxiedServerHeartbeat(t *testing.T) { if !expires.After(time.Now().UTC()) { t.Errorf("re-armed lease_expires_at should be in the future, got %v", expires) } + refreshOut, refreshErrOut, refreshErr := bdProxiedRunBuffers(t, bd, p.dir, "--actor", "hand-worker", "heartbeat", issue.ID) + if refreshErr != nil { + t.Fatalf("refresh heartbeat after the re-arm: %v\nstdout:\n%s\nstderr:\n%s", refreshErr, refreshOut, refreshErrOut) + } + if commitsRefresh := doltCommits(t); commitsRefresh != commitsAfter { + t.Errorf("refresh heartbeat minted Dolt commits: %d -> %d, want none", commitsAfter, commitsRefresh) + } }) } diff --git a/cmd/bd/heartbeat_proxied_server.go b/cmd/bd/heartbeat_proxied_server.go index c09772f52..efd9fedb2 100644 --- a/cmd/bd/heartbeat_proxied_server.go +++ b/cmd/bd/heartbeat_proxied_server.go @@ -7,49 +7,68 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" ) -// heartbeatProxiedOutcome carries what the render layer needs out of the -// transaction: the fully-resolved ID and the title for the feedback line. -type heartbeatProxiedOutcome struct { - id string - title string -} - -// runHeartbeatProxiedServer routes bd heartbeat through the proxied-server -// plane. The lease write is EPHEMERAL (the dolt_ignored leases table, -// bd-lrgn1): the transaction commits via uow.RunTxEphemeral's SQL-only form, -// so a heartbeat mints exactly ZERO Dolt commits per invocation — the same -// commit discipline as the classic DoltStore.HeartbeatIssue, and deliberately -// nothing here sets commandDidWrite or creates a Dolt commit. The exit code -// is the worker contract (workers call this every ~90s and only check rc): -// 0 = lease refreshed; nonzero = the lease is gone (wrong owner, not -// in_progress, closed, reclaimed) and the worker should stop. -func runHeartbeatProxiedServer(ctx context.Context, id string) error { - if uowProvider == nil { - return HandleErrorRespectJSON("proxied-server UOW provider not initialized") - } - - res, err := uow.RunTxEphemeral(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (heartbeatProxiedOutcome, error) { - issue, _, rerr := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) - if errors.Is(rerr, storage.ErrNotFound) { - return heartbeatProxiedOutcome{}, fmt.Errorf("issue %s not found", id) - } - if rerr != nil { - return heartbeatProxiedOutcome{}, fmt.Errorf("resolving %s: %w", id, rerr) - } - // Wisps resolve here too and are refused below: the repo verb - // classifies them ErrNotClaimable ("is ephemeral"), same as classic. - if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, currentActor()); herr != nil { - return heartbeatProxiedOutcome{}, fmt.Errorf("heartbeat %s: %w", issue.ID, herr) - } - return heartbeatProxiedOutcome{id: issue.ID, title: issue.Title}, nil - }) +// runHeartbeatProxied is bd heartbeat on the proxied-server plane: the id is +// resolved exactly, on either plane, in a read unit of work of its own, and +// the refresh goes through the provider's OWN Heartbeater accessor (the +// capability the uow leg commits with RunTxScoped: a refresh names no tables +// and takes its SQL-only form, so it mints exactly ZERO Dolt commits, +// bd-lrgn1; a re-arm mints at most one, none when a concurrent writer's Dolt +// commit already took its row). Nothing here sets commandDidWrite. +// The exit code is the worker contract: 0 = lease refreshed; nonzero = the +// lease is gone and the worker should stop. +func runHeartbeatProxied(ctx context.Context, id string) error { + issue, err := proxiedHeartbeatTarget(ctx, id) if err != nil { - return HandleErrorRespectJSON("%v", err) + return err } + heartbeater, err := proxiedHeartbeater() + if err != nil { + return HandleErrorRespectJSON("heartbeat %s: %v", issue.ID, err) + } + if _, err := heartbeater.Heartbeat(ctx, issueops.HeartbeatRequest{ + Actor: currentActor(), + IssueID: issue.ID, + }); err != nil { + return HandleErrorRespectJSON("heartbeat %s: %v", issue.ID, err) + } + SetLastTouchedID(issue.ID) + return renderHeartbeatSuccess(issue.ID, issue.Title) +} - SetLastTouchedID(res.id) - return renderHeartbeatSuccess(res.id, res.title) +// proxiedHeartbeatTarget reads the row a proxied heartbeat names, on either +// plane (a wisp resolves here and the role refuses it as ephemeral), for its +// canonical id and the title the feedback line prints. +func proxiedHeartbeatTarget(ctx context.Context, id string) (*types.Issue, error) { + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + issue, _, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + if errors.Is(err, storage.ErrNotFound) { + return nil, HandleErrorRespectJSON("issue %s not found", id) + } + if err != nil { + return nil, HandleErrorRespectJSON("resolving %s: %v", id, err) + } + return issue, nil +} + +// proxiedHeartbeater takes the lease refresh through the proxied-server +// provider's own capability accessor, proxiedLeaseReclaimer's two-step and for +// its reason: the accessor is where each layer is added. +func proxiedHeartbeater() (issueops.Heartbeater, error) { + if uowProvider == nil { + return nil, errors.New("proxied-server UOW provider not initialized") + } + src, ok := uowProvider.(uow.HeartbeaterSource) + if !ok { + return nil, fmt.Errorf("proxied-server provider %T does not offer the lease-refresh surface", uowProvider) + } + return src.Heartbeater() } diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index 3243a943e..099dd8948 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -320,6 +320,7 @@ func runServe() error { GraphCounter: roles.edgeCounter, BatchGetter: roles.batchGetter, LeaseReclaimer: roles.leaseReclaimer, + Heartbeater: roles.heartbeater, Relations: roles.relations, Commenter: roles.commenter, BlockingAnnotator: roles.blocking, @@ -682,6 +683,7 @@ type serveRoleSource interface { GraphCounter() (issueops.GraphCounter, error) BatchGetter() (issueops.BatchGetter, error) LeaseReclaimer() (issueops.LeaseReclaimer, error) + Heartbeater() (issueops.Heartbeater, error) IssueRelations() (issueops.Relations, error) Commenter() (issueops.Commenter, error) BlockingAnnotator() (issueops.BlockingAnnotator, error) @@ -750,6 +752,7 @@ func serveIssueRoles(src serveRoleSource, journalEnabled bool) (serveRoles, erro {"graph counter", func() (err error) { roles.edgeCounter, err = src.GraphCounter(); return }}, {"batch getter", func() (err error) { roles.batchGetter, err = src.BatchGetter(); return }}, {"lease reclaimer", func() (err error) { roles.leaseReclaimer, err = src.LeaseReclaimer(); return }}, + {"heartbeater", func() (err error) { roles.heartbeater, err = src.Heartbeater(); return }}, {"issue relations", func() (err error) { roles.relations, err = src.IssueRelations(); return }}, {"commenter", func() (err error) { roles.commenter, err = src.Commenter(); return }}, {"blocking annotator", func() (err error) { roles.blocking, err = src.BlockingAnnotator(); return }}, @@ -866,8 +869,12 @@ type serveRoles struct { // leaseReclaimer is the stale-lease sweep behind // POST /v0/beads/issues:reclaim. Its hook decorator WRAPS (a reclaim is an // update per reverted row), which is why it comes off the peeled store with - // the releaser: bd serve runs no hooks. + // the releaser: bd serve runs no hooks. heartbeater, the lease refresh + // behind POST /v0/beads/issues/{id}:heartbeat, comes off the same store + // for the same reason: its decorator WRAPS too, firing on_update for a + // re-arm (a refresh fires nothing). leaseReclaimer issueops.LeaseReclaimer + heartbeater issueops.Heartbeater relations issueops.Relations commenter issueops.Commenter blocking issueops.BlockingAnnotator diff --git a/cmd/bd/serve_source_test.go b/cmd/bd/serve_source_test.go index 1cb7ac105..f736bf2b2 100644 --- a/cmd/bd/serve_source_test.go +++ b/cmd/bd/serve_source_test.go @@ -101,6 +101,7 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { readyClaimer: &serveStubReadyClaimer{}, releaser: &serveStubReleaser{}, reclaimer: &serveStubLeaseReclaimer{}, + heartbeater: &serveStubHeartbeater{}, lifecycle: &serveStubLifecycle{}, dependencies: &serveStubDependencyEditor{}, batchApplier: &serveStubBatchApplier{}, @@ -267,6 +268,9 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.leaseReclaimer) || roles.leaseReclaimer != issueops.LeaseReclaimer(middle.reclaimer) { t.Errorf("lease reclaimer came from %p, want the layer directly beneath the hooks (%p)", roles.leaseReclaimer, middle.reclaimer) } + if storage.RoleFiresHooks(roles.heartbeater) || roles.heartbeater != issueops.Heartbeater(middle.heartbeater) { + t.Errorf("heartbeater came from %p, want the layer directly beneath the hooks (%p)", roles.heartbeater, middle.heartbeater) + } reader, claimer := roles.reader, roles.claimer // The same predicate httpapi.Listen refuses on, so a regression here is a // server that refuses to boot rather than one that runs hooks silently. @@ -420,6 +424,7 @@ type serveRolesStore struct { readyClaimer *serveStubReadyClaimer releaser *serveStubReleaser reclaimer *serveStubLeaseReclaimer + heartbeater *serveStubHeartbeater lifecycle *serveStubLifecycle dependencies *serveStubDependencyEditor batchApplier *serveStubBatchApplier @@ -480,6 +485,13 @@ func (s *serveRolesStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return s.reclaimer, nil } +// Heartbeater carries an identifiable value so the test can assert which layer +// bd serve took it from: the layer beneath the hooks, since the hook decorator +// wraps it (a re-arm fires on_update). +func (s *serveRolesStore) Heartbeater() (issueops.Heartbeater, error) { + return s.heartbeater, nil +} + // IssueLifecycle carries an identifiable value for the same reason the reader // and the claimer do: it is one of the roles the hook decorator wraps, so a peel // of the wrong depth would hand bd serve a lifecycle that runs the workspace's @@ -676,6 +688,12 @@ func (*serveStubReleaser) Release(context.Context, issueops.ReleaseRequest) (iss return issueops.ReleaseResult{}, errors.ErrUnsupported } +type serveStubHeartbeater struct{} + +func (*serveStubHeartbeater) Heartbeat(context.Context, issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + return issueops.HeartbeatResult{}, errors.ErrUnsupported +} + type serveStubLeaseReclaimer struct{} func (*serveStubLeaseReclaimer) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { diff --git a/cmd/bd/serve_store_identity_test.go b/cmd/bd/serve_store_identity_test.go index 4d3efc3f9..8a4f02474 100644 --- a/cmd/bd/serve_store_identity_test.go +++ b/cmd/bd/serve_store_identity_test.go @@ -217,6 +217,12 @@ func (*serveIdentityStore) Releaser() (issueops.Releaser, error) { return serveI func (*serveIdentityStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return serveIdentityRole{}, nil } + +// Heartbeater is declared at depth 1 for the same reason: its hook decorator +// recurses, and the recursion would otherwise land on a nil-embedded promotion. +func (*serveIdentityStore) Heartbeater() (issueops.Heartbeater, error) { + return serveIdentityRole{}, nil +} func (*serveIdentityStore) IssueLifecycle() (issueops.Lifecycle, error) { return serveIdentityRole{}, nil } @@ -305,6 +311,7 @@ type serveIdentityRole struct { issueops.Lifecycle issueops.Releaser issueops.LeaseReclaimer + issueops.Heartbeater issueops.ReadyClaimer issueops.BatchCloser issueops.MetadataCAS diff --git a/docs/multi-agent/federation.md b/docs/multi-agent/federation.md index 2bff9ccb8..fbdaccb1b 100644 --- a/docs/multi-agent/federation.md +++ b/docs/multi-agent/federation.md @@ -273,9 +273,10 @@ local heartbeat is keeping it alive: row from the snapshot, granting replica included — so this node's own stale lease can come back labelled with a remote one. -(The one path that moves a lease the other way is a heartbeat whose holder is a -different *spelling* of the lease row's holder: it re-arms through the upsert, -which stamps this node.) +(No heartbeat moves an existing lease, including one whose holder is a +different *spelling* of the lease row's holder: that row is refreshed in place +and keeps its granting replica. Only a re-arm, which grants a lease to a live +claim that had none, stamps this node.) Recover a stranded lease with `bd reclaim --any-replica`, once you have confirmed the granting replica is not still reaping — that confirmation is the diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 2f8fe6859..514bbcfe0 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "events.go", "events_watch.go", "handlers.go", + "heartbeat.go", "memories.go", "metadata_cas.go", "pinning.go", @@ -90,6 +91,7 @@ go_test( "events_watch_test.go", "gate_fields_test.go", "get_issue_test.go", + "heartbeat_test.go", "large_apply_client_budget_test.go", "large_batch_apply_test.go", "list_comments_omitted_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index e530e42a7..34d140240 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1119,7 +1119,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`, `dependencies.blocking.isBlocked`, which announces that `GET /v0/beads/dependencies/blocking` accepts `is_blocked` and answers each item's stored, transitive `is_blocked` flag (see that parameter) rather than silently answering `unknown_parameter`, and `issues.update.claim`, which announces that `PATCH /v0/beads/issues/{id}` accepts `claim` (see `UpdateIssueRequest`) and resolves the claimed id on both planes, so a wisp is claimable there, rather than a `400` naming the member unknown, and `issues.create.gates`, which announces that `POST /v0/beads/issues` and `issues:batchApply` create items accept the gate members `await_type`, `await_id`, `timeout` and `waiters` (see `CreateIssueRequest`), and `issues.update.gates`, which announces that `PATCH /v0/beads/issues/{id}`'s `patch` and an `issues:batchApply` update item's `patch` accept `await_id` and `waiters` (see `IssuePatchBody`), each rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.heartbeat`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`, `dependencies.blocking.isBlocked`, which announces that `GET /v0/beads/dependencies/blocking` accepts `is_blocked` and answers each item's stored, transitive `is_blocked` flag (see that parameter) rather than silently answering `unknown_parameter`, and `issues.update.claim`, which announces that `PATCH /v0/beads/issues/{id}` accepts `claim` (see `UpdateIssueRequest`) and resolves the claimed id on both planes, so a wisp is claimable there, rather than a `400` naming the member unknown, and `issues.create.gates`, which announces that `POST /v0/beads/issues` and `issues:batchApply` create items accept the gate members `await_type`, `await_id`, `timeout` and `waiters` (see `CreateIssueRequest`), and `issues.update.gates`, which announces that `PATCH /v0/beads/issues/{id}`'s `patch` and an `issues:batchApply` update item's `patch` accept `await_id` and `waiters` (see `IssuePatchBody`), each rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1497,6 +1497,30 @@ type Health struct { // HealthStatus defines model for Health.Status. type HealthStatus string +// HeartbeatIssueRequest defines model for HeartbeatIssueRequest. +type HeartbeatIssueRequest struct { + // Actor The lease holder refreshing its lease, and the fence's subject. `ClaimRequest.actor`'s rules exactly: the server trims it, then refuses an empty result, anything longer than 256 BYTES (or 255 characters, the column's bound), and any control character including newline. + Actor string `json:"actor"` +} + +// HeartbeatIssueResponse defines model for HeartbeatIssueResponse. +type HeartbeatIssueResponse struct { + // HeartbeatAt The server clock, in UTC, the refresh stamped on the lease. + HeartbeatAt time.Time `json:"heartbeat_at"` + + // Holder The lease holder after the refresh (the request's `actor`). + Holder string `json:"holder"` + + // Id The issue whose lease was refreshed. + Id string `json:"id"` + + // LeaseExpiresAt `heartbeat_at` plus the server's lease TTL: the deadline the next heartbeat must beat before a reclaim may revert the claim. + LeaseExpiresAt time.Time `json:"lease_expires_at"` + + // Rearmed True when the actor held the claim but no lease row existed for it (a claim hand-doled through `updateIssue`), so the heartbeat granted a fresh lease instead of refreshing one. A re-arm is recorded as a mutation of the issue: its `revision` is reminted, `updated_at` moves and an event is recorded, so re-read the issue before a guarded write. False for every refresh, which writes only the lease. + Rearmed bool `json:"rearmed"` +} + // Issue A tracked work item. Property semantics documented here apply to every schema that repeats them below. type Issue = types.Issue @@ -1702,7 +1726,7 @@ type Problem struct { // IT IS OPTIONAL. A refusal that could not name its blockers omits it and keeps the generic `detail`; absence means "this refusal did not name them", never "nothing blocks it". Re-read the issue's dependencies then. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `issue_pinned` (409), `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). + // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `lease_not_held` (409), `template_read_only` (409), `issue_pinned` (409), `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). Code string `json:"code"` // DeclaredLater With `invalid_argument` on a batch operation whose items may name each other: whether the unresolvable key IS declared by the request, at a LATER index. @@ -1772,7 +1796,7 @@ type Problem struct { // With `precondition_failed`: the body member carrying the guard that missed. It is the same spelling a 400 on the same operation would use, so a client reads one member to find the offending input whichever way the request was refused. Param *string `json:"param,omitempty"` - // Reason With `invalid_argument`: `unknown_parameter` (this server does not know that parameter — version skew; degrade or fall back), `invalid_value` (the value is not one this server will act on: malformed, out of vocabulary, or — for `limit=0` under `--allow-non-loopback` — legal but refused in this server's configuration; `detail` says which), `project_mismatch` (the `Bd-Project-Id` header named a project this server does not serve — a document-level refusal like the Host-header 400, raised on every enforced route, and the one that carries `server_project_id`; see the document-level rule), or `wire_revision_unsupported` (the `Bd-Wire-Revision` header named a revision below `ContextResponse.min_client_wire_revision` — also a document-level refusal, raised on every route including the identity handshake, and the one that carries `min_wire_revision`; see the document-level rule and `ContextResponse.wire_revision`). Either way the recovery is to send something different, never to retry the same request. The set may grow; default-branch on unknown values. + // Reason With `invalid_argument`: `unknown_parameter` (this server does not know that parameter — version skew; degrade or fall back), `invalid_value` (the value is not one this server will act on: malformed, out of vocabulary, or — for `limit=0` under `--allow-non-loopback` — legal but refused in this server's configuration; `detail` says which), `project_mismatch` (the `Bd-Project-Id` header named a project this server does not serve — a document-level refusal like the Host-header 400, raised on every enforced route, and the one that carries `server_project_id`; see the document-level rule), or `wire_revision_unsupported` (the `Bd-Wire-Revision` header named a revision below `ContextResponse.min_client_wire_revision` — also a document-level refusal, raised on every route including the identity handshake, and the one that carries `min_wire_revision`; see the document-level rule and `ContextResponse.wire_revision`). Either way the recovery is to send something different, never to retry the same request. With `lease_not_held`: `ephemeral` (the id names a wisp, which is never leased; its `issue_status` may be `in_progress`). The set may grow; default-branch on unknown values. Reason *string `json:"reason,omitempty"` // RequestId Opaque correlation id for this request, echoed in the server's request log line. Never a dispatch key and never a retry key. (This server mints per-process ids that do not survive a restart; a deployment may substitute any identifier with the same log-correlation property, such as an edge trace id.) @@ -2933,6 +2957,9 @@ type ClaimIssueJSONRequestBody = ClaimRequest // CloseIssueJSONRequestBody defines body for CloseIssue for application/json ContentType. type CloseIssueJSONRequestBody = CloseIssueRequest +// HeartbeatIssueJSONRequestBody defines body for HeartbeatIssue for application/json ContentType. +type HeartbeatIssueJSONRequestBody = HeartbeatIssueRequest + // ReleaseIssueJSONRequestBody defines body for ReleaseIssue for application/json ContentType. type ReleaseIssueJSONRequestBody = ReleaseIssueRequest diff --git a/internal/httpapi/claim.go b/internal/httpapi/claim.go index 92368aea7..18efc28ea 100644 --- a/internal/httpapi/claim.go +++ b/internal/httpapi/claim.go @@ -352,6 +352,7 @@ var ( _ uow.GraphCounterSource = timedProvider{} _ uow.BatchGetterSource = timedProvider{} _ uow.LeaseReclaimerSource = timedProvider{} + _ uow.HeartbeaterSource = timedProvider{} _ uow.RelationsSource = timedProvider{} _ uow.CommenterSource = timedProvider{} _ uow.BlockingAnnotatorSource = timedProvider{} @@ -482,6 +483,12 @@ func (p timedProvider) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return uow.NewLeaseReclaimer(p) } +// Heartbeater builds the lease refresh OVER THIS WRAPPER, for the same reason +// and with the same hazard as IssueReader. +func (p timedProvider) Heartbeater() (issueops.Heartbeater, error) { + return uow.NewHeartbeater(p) +} + // MoleculeStepper builds the advance-a-molecule role OVER THIS WRAPPER, so // every read and the claim it composes go through this wrapper's roles. func (p timedProvider) MoleculeStepper() (issueops.MoleculeStepper, error) { diff --git a/internal/httpapi/heartbeat.go b/internal/httpapi/heartbeat.go new file mode 100644 index 000000000..83fee48d3 --- /dev/null +++ b/internal/httpapi/heartbeat.go @@ -0,0 +1,178 @@ +package httpapi + +import ( + "errors" + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// heartbeatRequestMembers is the document's member list for +// HeartbeatIssueRequest. The schema is additionalProperties: false, so anything +// else is refused BY NAME, the posture every body on this surface takes. +var heartbeatRequestMembers = []string{claimActorMember} + +// handleHeartbeat refreshes the lease on one issue the actor holds: the verb +// `bd heartbeat` spells, behind issueops.Heartbeater. +// +// A REFRESH WRITES ONLY THE LEASE ROW, and that is the role's promise rather +// than this handler's: no revision is reminted (so the holder's own guarded +// writes keep their token across heartbeats), no version is recorded, +// updated_at does not move, and no event or journal entry is written. A +// RE-ARM (`rearmed: true`: a live claim that had no lease row) is recorded as +// a mutation and remints the revision; see issueops.Heartbeater. The response +// carries no `revision` member either way: it reports the lease, not the +// issue, so a refresh publishes no unchanged token a reader could take for a +// write, and `rearmed: true` tells the caller to re-read the issue before a +// guarded write. +// +// THE ACTOR IS THE FENCE'S SUBJECT, the release's posture: the heartbeat lands +// only while the actor holds the claim. It is caller-asserted provenance, not +// authenticated identity, exactly as on the claim. +// +// PLANES: the id resolves across both, so a wisp is answered with the role's +// typed refusal (`lease_not_held`) rather than a 404 — wisps are never leased. +func (s *Server) handleHeartbeat(w http.ResponseWriter, r *http.Request) { + // The custom-method dispatcher split the id off the segment and bounded it + // before this handler was chosen at all; see customMethodTarget. + id := r.PathValue(customMethodIDValue) + if !s.requireNoQuery(w, r) { + return + } + if !s.requireJSONContent(w, r) { + return + } + actor, ok := s.heartbeatActor(w, r) + if !ok { + return + } + + heartbeater, err := s.heartbeater(r) + if err != nil { + s.failHeartbeat(w, r, err) + return + } + result, err := heartbeater.Heartbeat(r.Context(), issueops.HeartbeatRequest{Actor: actor, IssueID: id}) + if err != nil { + s.failHeartbeat(w, r, err) + return + } + writeJSON(w, apigen.HeartbeatIssueResponse{ + Id: result.IssueID, + Holder: result.Holder, + HeartbeatAt: result.HeartbeatAt.UTC(), + LeaseExpiresAt: result.LeaseExpiresAt.UTC(), + Rearmed: result.Rearmed, + }) +} + +// heartbeatActor decodes the body and validates its one member by the claim's +// rules (validateActor, through bodyActor), and reports whether the request may +// proceed. Every refusal here happens BEFORE any database work. +func (s *Server) heartbeatActor(w http.ResponseWriter, r *http.Request) (string, bool) { + members, res := decodeJSONObjectBody(w, r) + if res != nil { + s.fail(w, r, *res) + return "", false + } + if offender, unknown := unknownMember(members, heartbeatRequestMembers); unknown { + s.failUnknownMember(w, r, offender, heartbeatRequestMembers) + return "", false + } + return s.bodyActor(w, r, members) +} + +// failHeartbeat answers a failed heartbeat, mapping the role's TYPED refusals +// onto the frozen codes. +// +// The 409 arms are matched first, from *issueops.LeaseConflictError's typed +// fields — never from the sentinel's prose — and BEFORE the generic arms: +// ErrLeaseNotHeld also matches ErrNotClaimable, which ClassifyError would +// answer `not_claimable`, the code CodeLeaseNotHeld's doc says this must not +// be. +// +// A FOREIGN HOLDER is the claim's own conflict shape: `already_claimed` +// naming the holder in `assignee`, with `issue_status`. The role reads both +// inside the refusing transaction. +func (s *Server) failHeartbeat(w http.ResponseWriter, r *http.Request, err error) { + // The 4xx path does not log by default, so the refusals whose real reason + // the response replaces with the server's own words are recorded here. + refused := func() { + s.event("request_refused", "request_id", requestInfo(r.Context()).id, "error", err.Error()) + } + + var conflict *issueops.LeaseConflictError + var field *issueops.HeartbeatFieldError + switch { + case errors.As(err, &conflict) && errors.Is(err, issueops.ErrAlreadyClaimed): + res := newResult(CodeAlreadyClaimed, + "another actor holds this issue's claim; nothing on the issue was written") + if conflict.Holder != "" { + res = res.WithAssignee(conflict.Holder) + } + if conflict.Status != "" { + res = res.WithIssueStatus(string(conflict.Status)) + } + s.fail(w, r, res) + + case errors.As(err, &conflict) && errors.Is(err, issueops.ErrLeaseNotHeld): + refused() + // The detail names the state the refusing transaction found, from the + // typed fields (never the role's prose): a wisp, an unclaimed issue, or + // the status the claim was ended into. A wisp also carries + // `reason: ephemeral`, the typed form a client reads. + res := newResult(CodeLeaseNotHeld, leaseNotHeldDetail(conflict)) + if conflict.Status != "" { + res = res.WithIssueStatus(string(conflict.Status)) + } + if conflict.Ephemeral { + res = res.WithReason(ReasonEphemeral) + } + s.fail(w, r, res) + + case errors.Is(err, storage.ErrNotFound): + s.fail(w, r, NotFound()) + + // The role's request rules. The actor's are refused at the edge by the + // claim's stricter rules, and the dispatcher guarantees a non-empty id, so + // this is defensive — but it names the member when it fires, rather than + // a generic refusal. + case errors.As(err, &field) && field.Field == issueops.HeartbeatFieldActor: + refused() + s.fail(w, r, InvalidArgument(claimActorMember, ReasonInvalidValue, field.Detail)) + + case !errors.Is(err, storage.ErrValidation): + s.failErr(w, r, err) + + default: + refused() + s.fail(w, r, InvalidArgument("", ReasonInvalidValue, + "the request was refused by this workspace's own validation; nothing was written")) + } +} + +// leaseNotHeldDetail says WHICH state refused a heartbeat with no live claim, +// from *issueops.LeaseConflictError's typed fields. Each sentence names one +// state only, so a reader of `bd heartbeat` output (and a pattern over it) +// can tell a closed issue from a released one and from a wisp. +func leaseNotHeldDetail(conflict *issueops.LeaseConflictError) string { + // The tail is "on the issue": the one write a refusal can make is deleting + // a stale lease row this actor still held (see the 409's description). + const tail = "; nothing on the issue was written" + switch { + case conflict.Ephemeral: + return "this issue is an ephemeral wisp, which is never leased" + tail + case conflict.Status == "": + return "this actor holds no live claim on this issue" + tail + case conflict.Status == types.StatusInProgress: + return "this actor holds no live claim on this issue: it is in_progress but not claimed by this actor" + tail + case conflict.Status == types.StatusOpen: + return "this actor holds no live claim on this issue: it is open and not claimed (released, reclaimed or never claimed)" + + tail + ". Claim it again rather than heartbeating" + default: + return "this actor holds no live claim on this issue: it is " + string(conflict.Status) + tail + } +} diff --git a/internal/httpapi/heartbeat_test.go b/internal/httpapi/heartbeat_test.go new file mode 100644 index 000000000..28acec2c7 --- /dev/null +++ b/internal/httpapi/heartbeat_test.go @@ -0,0 +1,283 @@ +package httpapi + +import ( + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// These cover the transport half of POST /v0/beads/issues/{id}:heartbeat: the +// body shape, the refusals the handler raises itself, how the role's typed +// refusals reach the wire, and the projection of the role's result. Everything +// below the wire (the lease-only write, the in_progress fence, the re-arm) is +// issueops.Heartbeater's, pinned by backend/conformance/heartbeater_contract.go +// on every leg, the served one included. + +const heartbeatPath = "/v0/beads/issues/bd-1:heartbeat" + +func newHeartbeatServer(t *testing.T, heartbeater *roleHeartbeater) *testServer { + t.Helper() + return newTestServer(t, rolesConfig(Config{Heartbeater: heartbeater})) +} + +// TestHeartbeatMapsTheRequestAndProjectsTheLease pins both halves of a +// success: the path id and the trimmed actor reach the role, and the role's +// result reaches the wire member for member — with NO `revision` and no issue +// body, because the response reports the lease, not the issue: even after a +// re-arm, which reminted the revision, `rearmed` is how the caller learns to +// re-read the issue. +func TestHeartbeatMapsTheRequestAndProjectsTheLease(t *testing.T) { + at := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) + heartbeater := &roleHeartbeater{result: issueops.HeartbeatResult{ + IssueID: "bd-1", + Holder: "worker", + HeartbeatAt: at, + LeaseExpiresAt: at.Add(5 * time.Minute), + Rearmed: true, + }} + ts := newHeartbeatServer(t, heartbeater) + + resp := ts.claim(t, heartbeatPath, `{"actor":" worker "}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + calls := heartbeater.heartbeatRequests() + if len(calls) != 1 { + t.Fatalf("role calls = %d, want 1", len(calls)) + } + if want := (issueops.HeartbeatRequest{Actor: "worker", IssueID: "bd-1"}); calls[0] != want { + t.Fatalf("role request = %+v, want %+v", calls[0], want) + } + + body := decodeBody(t, resp) + want := map[string]any{ + "id": "bd-1", + "holder": "worker", + "heartbeat_at": "2026-10-16T12:00:00Z", + "lease_expires_at": "2026-10-16T12:05:00Z", + "rearmed": true, + } + if len(body) != len(want) { + t.Errorf("response members = %v, want exactly %v", body, want) + } + for member, value := range want { + if body[member] != value { + t.Errorf("%s = %v, want %v", member, body[member], value) + } + } + for _, absent := range []string{"revision", "issue", "changed"} { + if _, present := body[absent]; present { + t.Errorf("response carries %q; a heartbeat answers the lease, not the issue", absent) + } + } +} + +// TestHeartbeatRefusesABadRequestBeforeTheRole pins the edge's own refusals: +// each is a 400 naming the offender, and none reaches the role. +func TestHeartbeatRefusesABadRequestBeforeTheRole(t *testing.T) { + for _, tc := range []struct { + name string + path string + body string + param string + }{ + {"a missing actor", heartbeatPath, `{}`, "actor"}, + {"a blank actor", heartbeatPath, `{"actor":" "}`, "actor"}, + {"a null actor", heartbeatPath, `{"actor":null}`, "actor"}, + {"a non-string actor", heartbeatPath, `{"actor":7}`, "actor"}, + {"an actor with a newline", heartbeatPath, `{"actor":"a\nb"}`, "actor"}, + {"an actor one character over the column", heartbeatPath, `{"actor":"` + strings.Repeat("a", types.MaxFieldLen+1) + `"}`, "actor"}, + {"an unknown member", heartbeatPath, `{"actor":"worker","ttl_seconds":60}`, "ttl_seconds"}, + {"an expected_version member", heartbeatPath, `{"actor":"worker","expected_version":"3"}`, "expected_version"}, + {"a query parameter", heartbeatPath + "?force=true", `{"actor":"worker"}`, "force"}, + } { + t.Run(tc.name, func(t *testing.T) { + heartbeater := &roleHeartbeater{} + ts := newHeartbeatServer(t, heartbeater) + resp := ts.claim(t, tc.path, tc.body) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) || body["param"] != tc.param { + t.Fatalf("problem = %v, want invalid_argument naming %q", body, tc.param) + } + if calls := heartbeater.heartbeatRequests(); len(calls) != 0 { + t.Fatalf("the role was called %d times for a request the edge refuses", len(calls)) + } + }) + } +} + +// TestHeartbeatAnswersTheRefusalsTheRoleRaises walks the role's typed refusals. +// +// The 409 arms are matched BEFORE the generic ones, and two rows here exist to +// pin that: ErrLeaseNotHeld also matches ErrNotClaimable, which the generic +// classifier would answer `not_claimable` — the code this operation must not +// answer — and a foreign holder must arrive in the claim's own conflict shape, +// naming the holder. +func TestHeartbeatAnswersTheRefusalsTheRoleRaises(t *testing.T) { + for _, tc := range []struct { + name string + err error + status int + code Code + param string + members map[string]any + absent []string + // detail must appear in a lease_not_held detail, and none of + // notDetail may: each refusal names its own state only. + detail string + notDetail []string + }{ + { + name: "a foreign holder", + err: &issueops.LeaseConflictError{ + IssueID: "bd-1", Holder: "alice", Status: types.StatusInProgress, + Err: fmt.Errorf("%w: bd-1 is held by alice", issueops.ErrAlreadyClaimed), + }, + status: http.StatusConflict, + code: CodeAlreadyClaimed, + members: map[string]any{"assignee": "alice", "issue_status": "in_progress"}, + }, + { + name: "no live claim (closed)", + err: &issueops.LeaseConflictError{ + IssueID: "bd-1", Status: types.StatusClosed, Err: issueops.ErrLeaseNotHeld, + }, + status: http.StatusConflict, + code: CodeLeaseNotHeld, + members: map[string]any{"issue_status": "closed"}, + absent: []string{"assignee", "reason"}, + detail: "it is closed", + notDetail: []string{"ephemeral", "open"}, + }, + { + name: "no live claim (released: open, unassigned)", + err: &issueops.LeaseConflictError{ + IssueID: "bd-1", Status: types.StatusOpen, Err: issueops.ErrLeaseNotHeld, + }, + status: http.StatusConflict, + code: CodeLeaseNotHeld, + members: map[string]any{"issue_status": "open"}, + absent: []string{"assignee", "reason"}, + detail: "not claimed", + notDetail: []string{"ephemeral", "closed"}, + }, + { + name: "a wisp", + err: &issueops.LeaseConflictError{ + IssueID: "bd-1", Status: types.StatusInProgress, Ephemeral: true, + Err: fmt.Errorf("%w: bd-1 is ephemeral", issueops.ErrLeaseNotHeld), + }, + status: http.StatusConflict, + code: CodeLeaseNotHeld, + members: map[string]any{"issue_status": "in_progress", "reason": string(ReasonEphemeral)}, + absent: []string{"assignee"}, + detail: "ephemeral wisp", + notDetail: []string{"closed", "open"}, + }, + { + name: "an absent id", + err: fmt.Errorf("%w: bd-1", issueops.ErrNotFound), + status: http.StatusNotFound, + code: CodeNotFound, + }, + { + name: "the role's own actor rule", + err: &issueops.HeartbeatFieldError{Field: issueops.HeartbeatFieldActor, Detail: "validation failed: actor is too long"}, + status: http.StatusBadRequest, + code: CodeInvalidArgument, + param: "actor", + }, + { + name: "some other validation refusal", + err: fmt.Errorf("%w: refused", issueops.ErrValidation), + status: http.StatusBadRequest, + code: CodeInvalidArgument, + }, + { + name: "a storage fault", + err: errors.New("dial tcp 10.0.0.1:3306: connection refused"), + status: http.StatusInternalServerError, + code: CodeInternal, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ts := newHeartbeatServer(t, &roleHeartbeater{err: tc.err}) + resp := ts.claim(t, heartbeatPath, `{"actor":"worker"}`) + if resp.StatusCode != tc.status { + t.Fatalf("status = %d, want %d: %s", resp.StatusCode, tc.status, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(tc.code) { + t.Fatalf("code = %v, want %s (%v)", body["code"], tc.code, body) + } + if tc.param != "" && body["param"] != tc.param { + t.Errorf("param = %v, want %q", body["param"], tc.param) + } + for member, value := range tc.members { + if body[member] != value { + t.Errorf("%s = %v, want %v", member, body[member], value) + } + } + for _, member := range tc.absent { + if _, present := body[member]; present { + t.Errorf("%s carries %q; it is documented only beside already_claimed", tc.code, member) + } + } + detail, _ := body["detail"].(string) + if strings.Contains(detail, "10.0.0.1") { + t.Errorf("detail leaks the storage error: %q", detail) + } + // The lease_not_held rows share the code, so the detail names the + // one state the role observed, and nothing else. + if tc.detail != "" && !strings.Contains(detail, tc.detail) { + t.Errorf("detail = %q, want it to say %q", detail, tc.detail) + } + for _, not := range tc.notDetail { + if strings.Contains(detail, not) { + t.Errorf("detail = %q names %q; it must name only the state the role observed", detail, not) + } + } + }) + } +} + +// TestHeartbeatDocumentsItsOwnConflictVocabulary pins the operation's code +// row from the direction that would make it wrong: `lease_not_held` is minted +// for the unheld case, so the claim's `not_claimable` must not be documented +// here, and the fence is the holder, so there is no precondition_failed. +func TestHeartbeatDocumentsItsOwnConflictVocabulary(t *testing.T) { + codes := operationCodes[OpHeartbeatIssue] + for _, want := range []Code{CodeAlreadyClaimed, CodeLeaseNotHeld, CodeNotFound, CodeInvalidArgument} { + if !containsCode(codes, want) { + t.Errorf("heartbeatIssue does not document %s", want) + } + } + for _, refused := range []Code{CodeNotClaimable, CodePreconditionFailed} { + if containsCode(codes, refused) { + t.Errorf("heartbeatIssue documents %s; see CodeLeaseNotHeld", refused) + } + } + if got := CodeLeaseNotHeld.Status(); got != http.StatusConflict { + t.Errorf("lease_not_held status = %d, want 409", got) + } +} + +// TestHeartbeatIsAdvertisedAsItsOwnCapability pins the token a client +// preflights on. +func TestHeartbeatIsAdvertisedAsItsOwnCapability(t *testing.T) { + for _, token := range Capabilities() { + if token == "issues.heartbeat" { + return + } + } + t.Fatalf("Capabilities() = %v, want it to advertise issues.heartbeat", Capabilities()) +} diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index fb71323f7..a47896822 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -113,6 +113,18 @@ const ( // distinction before then reads the row, which the operation description // tells it to do for a safety reason rather than a taste one. CodeNotReleasable Code = "not_releasable" + // CodeLeaseNotHeld is a heartbeat refused because the actor holds no live + // claim on the row: it is open, closed, released, reclaimed, moved off + // in_progress, or a wisp (wisps are never leased). The `issue_status` + // extension carries the status the refusing transaction observed, from + // *issueops.LeaseConflictError's typed field. + // + // It is MINTED rather than folded into CodeNotClaimable for + // CodeNotReleasable's reason: an open, unassigned row is the most claimable + // row a workspace has, and answering `not_claimable` about it would send a + // worker somewhere there is nothing to find. A FOREIGN holder is not this + // code: that is the claim's own `already_claimed`, naming the holder. + CodeLeaseNotHeld Code = "lease_not_held" // CodeNotClosable is close policy refusing an unforced close: open // children, or a live blocker. The open-children refusal carries the count // in the `open_children` extension member, read inside the refusing @@ -293,6 +305,7 @@ var codeStatus = map[Code]int{ CodeIssuePinned: http.StatusConflict, CodeNotAssignee: http.StatusConflict, CodeNotReleasable: http.StatusConflict, + CodeLeaseNotHeld: http.StatusConflict, CodeDependencyCycle: http.StatusConflict, CodeDependencyExists: http.StatusConflict, CodeAlreadyExists: http.StatusConflict, @@ -345,6 +358,12 @@ const ( // client built for a revision it no longer supports, never to retry the same // request. ReasonWireRevisionUnsupported Reason = "wire_revision_unsupported" + // ReasonEphemeral qualifies a 409 CodeLeaseNotHeld: the id names a wisp, + // which is never leased, so no claim on it can ever be heartbeaten. It is + // the typed form of issueops.LeaseConflictError.Ephemeral, so a client + // tells a wisp from an unclaimed issue without parsing `detail` (a wisp's + // `issue_status` can be in_progress). + ReasonEphemeral Reason = "ephemeral" ) // staticDetail is the set of codes whose `detail` is FIXED, whatever the @@ -505,6 +524,13 @@ const ( // row, and this one sweeps MANY leases on a clock's say-so and reports an id // that was not a stale lease by leaving it out of `reclaimed`. OpReclaimIssues = "reclaimIssues" + // OpHeartbeatIssue refreshes the lease on one issue the actor holds, behind + // issueops.Heartbeater: what `bd heartbeat` spells. It is NOT claimIssue + // repeated: a re-claim is idempotent for the holder WITHOUT extending the + // lease, and this extends the lease. A refresh writes nothing else — no + // revision, no version, no updated_at, no event; a re-arm (`rearmed`) is + // recorded as a mutation and writes each of them. + OpHeartbeatIssue = "heartbeatIssue" // OpListRelatedIssues reads ONE issue's neighbors in a named direction, // behind issueops.Relations. It is NOT listDependencies narrowed to one // anchor: that operation answers the stored edge ROWS with their targets @@ -764,6 +790,17 @@ var operationCodes = map[string][]Code{ // named a version of. Its only 400s are the role's own refusals and the // handler's shape checks. OpReclaimIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, + // Two 409s, and they answer different questions. `already_claimed` is a + // live FOREIGN holder, in the claim's own shape: `assignee` names the + // holder and `issue_status` the status, both from the refusing + // transaction's *issueops.LeaseConflictError. `lease_not_held` is the + // actor holding no live claim at all, with `issue_status`. No + // precondition_failed: the fence is the holder, not a revision. + OpHeartbeatIssue: { + CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, + CodeAlreadyClaimed, CodeLeaseNotHeld, + CodeBusy, CodeDBUnavailable, CodeInternal, + }, // The same vocabulary as the stored-edge read beside it, and no not_found // for a stronger version of the same reason: this operation probes no id's // existence at all, so there is nothing it could 404 on. @@ -1148,6 +1185,15 @@ func (r Result) WithIssueStatus(status string) Result { return r } +// WithReason attaches the `reason` member to a refusal other than the 400 +// (InvalidArgument sets its own). Same rule as WithAssignee: it comes from the +// typed error's own field, never from parsing the refusal's prose. +func (r Result) WithReason(reason Reason) Result { + v := string(reason) + r.Problem.Reason = &v + return r +} + // WithOpenChildren attaches the `open_children` extension member (how many open // children the transaction that refused a close observed). Same rule as // WithAssignee: it comes from the typed error's own field, read inside that diff --git a/internal/httpapi/roles_test.go b/internal/httpapi/roles_test.go index 6ada7952d..00fe4370f 100644 --- a/internal/httpapi/roles_test.go +++ b/internal/httpapi/roles_test.go @@ -190,6 +190,34 @@ func (l *roleLeaseReclaimer) reclaimRequests() []issueops.ReclaimRequest { return append([]issueops.ReclaimRequest(nil), l.calls...) } +// roleHeartbeater is the lease refresh of the store-shaped source, its own fake +// beside roleClaimer for roleBatchGetter's reason: a separate role on a +// separate accessor, so one fake answering both would let a test pass on a +// server that had wired the heartbeat handler to the claim. +type roleHeartbeater struct { + result issueops.HeartbeatResult + err error + + mu sync.Mutex + calls []issueops.HeartbeatRequest +} + +func (h *roleHeartbeater) Heartbeat(_ context.Context, req issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + h.mu.Lock() + h.calls = append(h.calls, req) + h.mu.Unlock() + if h.err != nil { + return issueops.HeartbeatResult{}, h.err + } + return h.result, nil +} + +func (h *roleHeartbeater) heartbeatRequests() []issueops.HeartbeatRequest { + h.mu.Lock() + defer h.mu.Unlock() + return append([]issueops.HeartbeatRequest(nil), h.calls...) +} + // roleRelations is the single-anchor NEIGHBOR role of the store-shaped source, // its own fake beside roleEdgeReader for roleGraphCounter's reason: the two sit // on adjacent accessors and answer about the same edges, so one fake serving @@ -991,6 +1019,9 @@ func rolesConfig(cfg Config) Config { if cfg.LeaseReclaimer == nil { cfg.LeaseReclaimer = &roleLeaseReclaimer{} } + if cfg.Heartbeater == nil { + cfg.Heartbeater = &roleHeartbeater{} + } if cfg.Relations == nil { cfg.Relations = &roleRelations{} } diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index b3ea05bc9..cfcb3aaf2 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -835,6 +835,18 @@ var routeTable = []route{ implemented: true, handler: (*Server).handleRelease, }, + { + op: OpHeartbeatIssue, + method: http.MethodPost, + // The lease refresh, on the dispatcher the close built. Everything the + // claim row says about the wildcard's width holds here unchanged. + pattern: customMethodPattern, + specPath: "/v0/beads/issues/{id}:heartbeat", + customMethod: ":heartbeat", + capability: "issues.heartbeat", + implemented: true, + handler: (*Server).handleHeartbeat, + }, { op: OpCloseIssue, method: http.MethodPost, diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index c4e72600a..64f814f20 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -293,6 +293,13 @@ type Config struct { // claim on its holder's say-so, and this reverts MANY on a clock's. // Required on the same terms as every field here. LeaseReclaimer issueops.LeaseReclaimer + // Heartbeater is the lease refresh behind + // POST /v0/beads/issues/{id}:heartbeat. It is a SEPARATE field from + // Claimer for the reason issueops.Heartbeater's own doc gives: a claim + // takes ownership and never extends the lease, a heartbeat proves the + // holder is alive and writes only the lease row. Required on the same + // terms as every field here. + Heartbeater issueops.Heartbeater // Relations is the single-anchor neighbor read behind // GET /v0/beads/issues/{id}/related. It is a SEPARATE field from EdgeReader // for the reason issueops.EdgeReader's own doc gives at length: that role @@ -442,6 +449,7 @@ type Server struct { issueEdgeCounter issueops.GraphCounter issueBatchGetter issueops.BatchGetter issueReclaimer issueops.LeaseReclaimer + issueHeartbeater issueops.Heartbeater issueRelations issueops.Relations issueCommenter issueops.Commenter issueBlocking issueops.BlockingAnnotator @@ -655,6 +663,7 @@ func Listen(cfg Config) (*Server, error) { issueEdgeCounter: cfg.GraphCounter, issueBatchGetter: cfg.BatchGetter, issueReclaimer: cfg.LeaseReclaimer, + issueHeartbeater: cfg.Heartbeater, issueRelations: cfg.Relations, issueCommenter: cfg.Commenter, issueBlocking: cfg.BlockingAnnotator, @@ -777,12 +786,12 @@ func Listen(cfg Config) (*Server, error) { // "all or nothing" would turn an honest condition into a special case inside // three functions. It is checked once, on its own, below. func sourceRoles(cfg Config) []any { - return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.LeaseReclaimer, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} + return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.LeaseReclaimer, cfg.Heartbeater, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} } // roleSourceNames spells sourceRoles for the refusal message, in the same // order, so a caller reading the error learns the whole set it must pass. -const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, LeaseReclaimer, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" +const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, LeaseReclaimer, Heartbeater, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" func anyRoleSet(cfg Config) bool { return slices.ContainsFunc(sourceRoles(cfg), func(r any) bool { return r != nil }) @@ -1184,6 +1193,18 @@ func (s *Server) leaseReclaimer(r *http.Request) (issueops.LeaseReclaimer, error return src.LeaseReclaimer() } +// heartbeater returns the lease-refresh surface for one request, on the same +// terms as every role above and held by INTERFACE so uow.HeartbeaterSource is +// load-bearing rather than decorative. HeartbeatResult is a plain value, so it +// goes out unwrapped. +func (s *Server) heartbeater(r *http.Request) (issueops.Heartbeater, error) { + if s.provider == nil { + return s.issueHeartbeater, nil + } + var src uow.HeartbeaterSource = timedProvider{inner: s.provider, rec: requestInfo(r.Context())} + return src.Heartbeater() +} + // relations returns the single-anchor neighbor surface for one request, built // the same two ways as edgeReader above and held by INTERFACE so // uow.RelationsSource is load-bearing rather than decorative. diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 809436592..4596a1111 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -3256,6 +3256,106 @@ paths: '503': $ref: '#/components/responses/Unavailable' + /v0/beads/issues/{id}:heartbeat: + post: + operationId: heartbeatIssue + summary: Refresh the claim lease on an issue the actor holds + description: >- + What `bd heartbeat` spells. It pushes the lease forward by the + server's lease TTL and stamps `heartbeat_at`, and a refresh writes ONLY + the lease. The issue row is untouched: `revision` is NOT reminted (a + following `expected_version` guard written against the pre-heartbeat + revision still wins), the versioned-history revision does not advance, + `updated_at` does not move, and no event or events-journal entry is + written (`events.list` / `events.watch` see nothing). + + + FENCED ON THE HOLDER AND THE LIVE CLAIM: `actor` must hold the claim, + and the issue must be `in_progress` under it when the lease is + refreshed. A holder that is `in_progress` with no lease row (a claim + hand-doled through `updateIssue`) is re-armed and reported + `rearmed: true`; a refresh of an existing lease row never is. A re-arm + is recorded as a mutation, a refresh is not: the re-arm remints + `revision`, moves `updated_at` and records an event (once, so that it + collides with a release or close racing it rather than leaving a + lease on an issue that is no longer claimed); re-read the issue + before a guarded write after `rearmed: true`. + `heartbeat_at` and `lease_expires_at` are whole seconds, the values + stored. Wisps hold no + lease and are refused. Safe to repeat: a repeat only moves the expiry + forward. The request carries no TTL, as the claim carries none: lease + policy is the server's, and `lease_expires_at` reports the result so a + worker can see its margin. + + + There is no `expected_version` member, deliberately: the fence is the + holder, and the revision changes on unrelated edits by the holder + itself, so a revision-fenced heartbeat would fail spuriously. + + + Available when the server advertises the `issues.heartbeat` capability. + parameters: + - $ref: '#/components/parameters/IssueID' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/HeartbeatIssueRequest' + security: + - bearerToken: [] + responses: + '200': + description: >- + The lease is refreshed. The body reports the lease, not the issue, + so there is no row and no `revision` in it. A refresh wrote nothing + to the issue row; after `rearmed: true` the issue's `revision` was + reminted, so re-read the issue before a guarded write. + content: + application/json: + schema: + $ref: '#/components/schemas/HeartbeatIssueResponse' + '400': + description: >- + Invalid request: an unknown query parameter, an unparseable or + oversized body, an unknown body member, or an `actor` refused by the + rules `ClaimRequest.actor` states. + x-bd-codes: [invalid_argument] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '401': + $ref: '#/components/responses/Unauthenticated' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: >- + `already_claimed` is a live FOREIGN holder, in the claim's own + shape: the `assignee` extension member names the holder and + `issue_status` carries the status, both read inside the refusing + transaction. + + + `lease_not_held` is the actor holding no live claim at all: the + issue is not `in_progress` under this actor (open, closed, + released, reclaimed, or moved to another status), or it is a wisp. + It carries `issue_status`, and `reason: ephemeral` for a wisp; + `detail` names the one state the refusing transaction found. The + recovery is a claim, not another heartbeat. Nothing on the issue + is written on either refusal; the one write a refusal can make is + deleting a stale lease row this actor still held on an issue it no + longer holds a live claim on (never another holder's lease). + x-bd-codes: [already_claimed, lease_not_held] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + /v0/beads/issues/{id}:close: post: operationId: closeIssue @@ -8262,6 +8362,7 @@ components: `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, + `issues.heartbeat`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, @@ -11186,6 +11287,55 @@ components: nothing — the idempotent re-claim. A claim held by a DIFFERENT actor is a 409, not a 200 with this flag. + HeartbeatIssueRequest: + type: object + additionalProperties: false + required: [actor] + properties: + actor: + type: string + minLength: 1 + maxLength: 256 + pattern: '^[^\u0000-\u001F\u007F-\u009F\u2028\u2029]+$' + description: >- + The lease holder refreshing its lease, and the fence's subject. + `ClaimRequest.actor`'s rules exactly: the server trims it, then + refuses an empty result, anything longer than 256 BYTES (or 255 + characters, the column's bound), and any control character + including newline. + + HeartbeatIssueResponse: + type: object + additionalProperties: false + required: [id, holder, heartbeat_at, lease_expires_at, rearmed] + properties: + id: + type: string + description: The issue whose lease was refreshed. + holder: + type: string + description: The lease holder after the refresh (the request's `actor`). + heartbeat_at: + type: string + format: date-time + description: The server clock, in UTC, the refresh stamped on the lease. + lease_expires_at: + type: string + format: date-time + description: >- + `heartbeat_at` plus the server's lease TTL: the deadline the next + heartbeat must beat before a reclaim may revert the claim. + rearmed: + type: boolean + description: >- + True when the actor held the claim but no lease row existed for it + (a claim hand-doled through `updateIssue`), so the heartbeat granted + a fresh lease instead of refreshing one. A re-arm is recorded as a + mutation of the issue: its `revision` is reminted, `updated_at` + moves and an event is recorded, so re-read the issue before a + guarded write. False for every refresh, which writes only the + lease. + ReleaseIssueRequest: type: object additionalProperties: false @@ -12731,6 +12881,7 @@ components: configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), + `lease_not_held` (409), `template_read_only` (409), `issue_pinned` (409), `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), @@ -12786,7 +12937,9 @@ components: and the one that carries `min_wire_revision`; see the document-level rule and `ContextResponse.wire_revision`). Either way the recovery is to send something different, never to retry - the same request. The set may grow; default-branch on unknown + the same request. With `lease_not_held`: `ephemeral` (the id + names a wisp, which is never leased; its `issue_status` may be + `in_progress`). The set may grow; default-branch on unknown values. assignee: type: string diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 53d2c7e57..1c04f4f95 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -2039,6 +2039,44 @@ ], "required": true }, + { + "schema": "HeartbeatIssueRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "HeartbeatIssueResponse", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "HeartbeatIssueResponse", + "member": "holder", + "type": "string", + "required": true + }, + { + "schema": "HeartbeatIssueResponse", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "HeartbeatIssueResponse", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "HeartbeatIssueResponse", + "member": "rearmed", + "type": "boolean", + "required": true + }, { "schema": "Issue", "member": "acceptance_criteria", @@ -5751,6 +5789,13 @@ "explode": true, "default": "false" }, + { + "schema": "param:heartbeatIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, { "schema": "param:listBlockingAnnotations", "member": "query:is_blocked", @@ -6404,6 +6449,8 @@ "EventRecord.issue": "response", "EventsPage": "response", "Health": "response", + "HeartbeatIssueRequest": "request", + "HeartbeatIssueResponse": "response", "Issue": "response", "Issue.labels[]": "response", "Issue.metadata": "response", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index d94cce698..20dace40a 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -27,6 +27,7 @@ go_library( "dial.go", "dispatch.go", "errors.go", + "heartbeater.go", "insecure_credential_guard.go", "journal.go", "leasereclaimer.go", @@ -119,6 +120,7 @@ go_test( "expected_revision_contract_test.go", "gate_fields_test.go", "getissue_miss_test.go", + "heartbeater_test.go", "helpers_test.go", "insecure_credential_guard_test.go", "lifecycle_create_test.go", @@ -154,6 +156,7 @@ go_test( "served_graph_aux_test.go", "served_graph_counter_test.go", "served_harness_test.go", + "served_heartbeater_test.go", "served_hooks_test.go", "served_issue_operations_test.go", "served_journal_test.go", diff --git a/internal/httpclient/accessors.go b/internal/httpclient/accessors.go index 8a6298155..4bb66f0f3 100644 --- a/internal/httpclient/accessors.go +++ b/internal/httpclient/accessors.go @@ -393,3 +393,12 @@ func (s *Store) BatchGetter() (issueops.BatchGetter, error) { func (s *Store) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return &httpLeaseReclaimer{store: s}, nil } + +// Heartbeater serves heartbeatIssue (POST /v0/beads/issues/{id}:heartbeat) — +// the lease refresh behind `bd heartbeat`. Like +// LeaseReclaimer it dials through the store directly rather than through +// roleWire: Store.dispatch preflights issues.heartbeat, so an older server is +// refused before the dial naming the token. See heartbeater.go. +func (s *Store) Heartbeater() (issueops.Heartbeater, error) { + return &httpHeartbeater{store: s}, nil +} diff --git a/internal/httpclient/completeness_test.go b/internal/httpclient/completeness_test.go index d8b06ba89..454694c77 100644 --- a/internal/httpclient/completeness_test.go +++ b/internal/httpclient/completeness_test.go @@ -153,9 +153,10 @@ var legitimatelyUnsupported = map[string]string{ // now, kept unwired for CreateIssue's rule rather than for the absence of an // operation. What has not moved is the LEASE row the release deletes: the // role owns that as part of its post-state, and neither of these front doors - // can say anything about it, which is why the heartbeat and the reaper above - // keep the original reason. - "HeartbeatIssue": "lease: no wire vocabulary", + // can say anything about it, which is why the raw reaper keeps the original + // reason. The HEARTBEAT has left this list: issues.heartbeat publishes the + // lease refresh and both Heartbeater and the legacy + // HeartbeatIssue dial it (heartbeater.go). "ReclaimExpiredLeases": "lease: no wire vocabulary", "UnclaimIssue": "write: raw release, force flag and all (role: Releaser). Re-routing it would put a second spelling of the transition beside the role's, with a bare error where the role answers the post-state row and its reminted version", "UnclaimIssueIfAssignee": "write: raw conditional release (role: Releaser, via ReleaseRequest.ExpectedAssignee). Its expectation is a STRING where the role's is a POINTER, so it cannot express the one distinction the role's guard is built on — absent means 'do not check' and empty is a refusal, and this signature spells both as \"\"", diff --git a/internal/httpclient/dispatch.go b/internal/httpclient/dispatch.go index eefc145e2..1946bf83e 100644 --- a/internal/httpclient/dispatch.go +++ b/internal/httpclient/dispatch.go @@ -38,10 +38,12 @@ const ( // // The bounded 503 retry lives here too: the wire layer bounds Retry-After and // hands it back but never sleeps on it. Whether a 503 is safe to come back to -// is the question wire.Request.Replayable answers, and only a read's answer -// is yes. (The wire does resend a replayable request once, at once, when its -// first attempt stalled before any headers or had its connection reset; that -// retry answers a transport-level failure, a 503 is the server's answer.) +// is the question wire.Request.Replayable answers: yes for a read, and for the +// one write marked Idempotent, the lease heartbeat, whose replay moves the +// expiry and applies nothing twice. (The wire does resend a replayable request +// once, at once, when its first attempt stalled before any headers or had its +// connection reset; that retry answers a transport-level failure, a 503 is the +// server's answer.) // // A store with no transport is a build-wiring fault (ErrNoTransport), not the // user's: the workspace and the server may both be fine. @@ -59,8 +61,9 @@ func (s *Store) dispatch(ctx context.Context, req wire.Request, out any) error { } err := s.wire.Do(ctx, req, out) - // Only a replayable read is retried: a GET, or a read on POST that says so - // (the batch get). A 503 can interrupt a non-idempotent write whose first + // Only a replayable request is retried: a GET, a read on POST that says so + // (the batch get), or the heartbeat, the one write that says so (see + // httpHeartbeater). A 503 can interrupt a non-idempotent write whose first // attempt already committed before the server ran out of slots, so // resending a POST/PATCH/DELETE could double-apply it — DELETE included, // since forgetMemory is a mutation the wire happens to spell as one. diff --git a/internal/httpclient/encode/oracle_test.go b/internal/httpclient/encode/oracle_test.go index f65bb152a..3a2d4891b 100644 --- a/internal/httpclient/encode/oracle_test.go +++ b/internal/httpclient/encode/oracle_test.go @@ -72,6 +72,7 @@ func startOracle(t *testing.T) *oracle { GraphCounter: stubGraphCounter{}, BatchGetter: stubBatchGetter{}, LeaseReclaimer: stubLeaseReclaimer{}, + Heartbeater: stubHeartbeater{}, Relations: stubRelations{}, Commenter: stubCommenter{}, BlockingAnnotator: stubBlocking{}, @@ -272,6 +273,7 @@ type ( stubGraphCounter struct{ issueops.GraphCounter } stubBatchGetter struct{ issueops.BatchGetter } stubLeaseReclaimer struct{ issueops.LeaseReclaimer } + stubHeartbeater struct{ issueops.Heartbeater } stubRelations struct{ issueops.Relations } stubCommenter struct{ issueops.Commenter } stubBlocking struct{ issueops.BlockingAnnotator } diff --git a/internal/httpclient/heartbeater.go b/internal/httpclient/heartbeater.go new file mode 100644 index 000000000..4fb90c693 --- /dev/null +++ b/internal/httpclient/heartbeater.go @@ -0,0 +1,159 @@ +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// httpHeartbeater is issueops.Heartbeater over heartbeatIssue (POST +// /v0/beads/issues/{id}:heartbeat), the lease refresh behind `bd heartbeat`. +// +// THE MAPPING IS TOTAL: the id rides in the path and the actor is the body's one +// member, so nothing here refuses on shape and the ledger carries no row for this +// operation. The answer carries no revision, deliberately: it reports the +// lease, not the issue. A refresh writes only the lease, so there is no token +// to stitch back; a re-arm remints it, which Rearmed reports so the caller +// re-reads the issue before a guarded write. +// +// THE REQUEST RULES ARE CHECKED BEFORE THE DIAL, in the order +// internal/storage/issueops.ValidateHeartbeatRequest applies them and with its +// *issueops.HeartbeatFieldError, for leasereclaimer.go's reason: the role's +// contract promises the typed field error, and a round trip would spend a write +// slot to be told what the request already said. The rules are restated rather +// than imported (that package is the storage engine's shared body); +// TestHeartbeatValidationMatchesTheSharedValidator pins the two together. +// +// AN OLDER SERVER IS REFUSED BEFORE THE DIAL. Store.dispatch preflights +// issues.heartbeat, so a server that does not advertise it answers +// *wire.CapabilityError naming the token (taxonomy case 2), never a fallback +// to a re-claim or a metadata stamp: a re-claim by the holder does not extend +// the lease, and a stamp would remint the revision the holder's guarded +// writes depend on. +// +// IT IS REPLAYABLE (wire.Request.Idempotent), the one write this client marks +// so. A repeat only moves the expiry later and writes nothing else (a repeat +// of a re-arm finds the lease row the first one granted and refreshes it), and +// a replay that lands after a reclaim fails the holder fence, which is the +// answer the worker needs anyway. So a 503 is retried within dispatch's bound +// rather than reported to a worker that heartbeats every ~90s and stops on any +// nonzero exit. +// +// THE COST: A REPLAY CAN UNDER-REPORT Rearmed. When a re-arm commits but its +// answer is lost (the wire resends after a stalled or reset attempt, dispatch +// after a 503), the replay finds the lease row the re-arm granted and answers +// a refresh, Rearmed false, though the revision token was reminted. The +// backstop is the guarded write itself: one composed on the old revision fails +// its revision check once and re-reads (see issueops.HeartbeatResult.Rearmed). +type httpHeartbeater struct{ store *Store } + +var _ issueops.Heartbeater = (*httpHeartbeater)(nil) + +// Heartbeat dials POST /v0/beads/issues/{id}:heartbeat. +// +// THE REFUSALS reach the caller typed as the role lists them: not_found is +// issueops.ErrNotFound, lease_not_held is *issueops.LeaseConflictError +// wrapping ErrLeaseNotHeld (the wire's code table builds it), and +// already_claimed — which the wire's table rebuilds as the CLAIM's conflict, +// the code's meaning on every other operation — is re-typed here as the +// heartbeat's *LeaseConflictError naming the holder (heartbeatError). +func (h *httpHeartbeater) Heartbeat(ctx context.Context, req issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + if err := validateHeartbeatRequest(req); err != nil { + return issueops.HeartbeatResult{}, err + } + path, err := wire.IssueMethodPath(req.IssueID, wire.MethodHeartbeat) + if err != nil { + return issueops.HeartbeatResult{}, err + } + + var out apigen.HeartbeatIssueResponse + if err := h.store.dispatch(ctx, wire.Request{ + Op: wire.OpHeartbeatIssue, + Method: http.MethodPost, + Path: path, + Body: apigen.HeartbeatIssueRequest{Actor: req.Actor}, + IssueID: req.IssueID, + Actor: req.Actor, + Idempotent: true, + }, &out); err != nil { + return issueops.HeartbeatResult{}, heartbeatError(err) + } + return issueops.HeartbeatResult{ + IssueID: out.Id, + Holder: out.Holder, + HeartbeatAt: out.HeartbeatAt.UTC(), + LeaseExpiresAt: out.LeaseExpiresAt.UTC(), + Rearmed: out.Rearmed, + }, nil +} + +// heartbeatError re-types the two refusals the shared problem table cannot +// spell as the heartbeat's. +// +// - already_claimed arrives as *issueops.ClaimConflictError (the claim's +// shape, from the same code). On this operation it is a foreign holder, +// so it becomes *issueops.LeaseConflictError naming that holder and +// wrapping ErrAlreadyClaimed, read in the native heartbeat's own words +// ("issue already claimed by "). +// - invalid_argument on `actor` is the server's actor rule (the claim's, +// which also refuses control characters) and becomes the role's +// *HeartbeatFieldError on that field. +// +// Everything else — lease_not_held, not_found, a capability refusal, a +// transport failure — is already the role's answer and travels unchanged. +func heartbeatError(err error) error { + var claim *issueops.ClaimConflictError + if errors.As(err, &claim) && errors.Is(err, issueops.ErrAlreadyClaimed) { + refusal := issueops.ErrAlreadyClaimed + if claim.Assignee != "" { + refusal = fmt.Errorf("%w by %s", issueops.ErrAlreadyClaimed, claim.Assignee) + } + return &issueops.LeaseConflictError{ + IssueID: claim.IssueID, Holder: claim.Assignee, Status: claim.Status, Err: refusal, + } + } + var problem *wire.ProblemError + if errors.As(err, &problem) && problem.Code == "invalid_argument" && problem.Param == "actor" { + return &issueops.HeartbeatFieldError{Field: issueops.HeartbeatFieldActor, Detail: problem.Detail} + } + return err +} + +// validateHeartbeatRequest restates the role's request rules, in its order: +// the actor present (blank once trimmed is absent), the actor within its +// column AS SENT (it is never trimmed: it is compared with the lease holder +// verbatim), then a non-empty id. +func validateHeartbeatRequest(req issueops.HeartbeatRequest) error { + field := func(name, detail string) error { + return &issueops.HeartbeatFieldError{Field: name, Detail: issueops.ErrValidation.Error() + ": " + detail} + } + if strings.TrimSpace(req.Actor) == "" { + return field(issueops.HeartbeatFieldActor, "heartbeat actor is required") + } + if err := types.CheckFieldLen("actor", req.Actor); err != nil { + return field(issueops.HeartbeatFieldActor, err.Error()) + } + if req.IssueID == "" { + return field(issueops.HeartbeatFieldIssueID, "heartbeat issue id is required") + } + return nil +} + +// HeartbeatIssue is the legacy lease refresh, served over the role so an +// in-process library caller holding a storage.DoltStorage reaches the same +// operation `bd heartbeat` does, with the same refusals. +func (s *Store) HeartbeatIssue(ctx context.Context, id, actor string) error { + role, err := s.Heartbeater() + if err != nil { + return err + } + _, err = role.Heartbeat(ctx, issueops.HeartbeatRequest{Actor: actor, IssueID: id}) + return err +} diff --git a/internal/httpclient/heartbeater_test.go b/internal/httpclient/heartbeater_test.go new file mode 100644 index 000000000..0c7029a1b --- /dev/null +++ b/internal/httpclient/heartbeater_test.go @@ -0,0 +1,372 @@ +package httpclient + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// The unit half of the Heartbeater client leg: the +// preflight, the request rules, the problem mapping and the replay, over the +// real wire client against a doctored server. The contract itself runs against +// a real bd serve in served_heartbeater_test.go. + +// heartbeatStub is a doctored bd serve: a handshake advertising caps, and a +// :heartbeat route that answers each POST with the next canned reply. +type heartbeatStub struct { + mu sync.Mutex + posts []string // the raw bodies, in arrival order + paths []string + replies []func(http.ResponseWriter) +} + +func (h *heartbeatStub) store(t *testing.T, caps ...string) *Store { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == wire.PathContext { + ctx := v0Context("proj-hb") + ctx.Capabilities = caps + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(ctx) + return + } + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, wire.MethodHeartbeat) { + http.NotFound(w, r) + return + } + body, _ := io.ReadAll(r.Body) + h.mu.Lock() + h.posts = append(h.posts, string(body)) + h.paths = append(h.paths, r.URL.Path) + var reply func(http.ResponseWriter) + if len(h.replies) > 0 { + reply, h.replies = h.replies[0], h.replies[1:] + } + h.mu.Unlock() + if reply == nil { + t.Errorf("an unexpected heartbeat POST reached the server: %s", body) + w.WriteHeader(http.StatusTeapot) + return + } + reply(w) + })) + t.Cleanup(srv.Close) + base, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + client, err := wire.New(base, nil, wire.Options{HTTPClient: srv.Client()}) + if err != nil { + t.Fatal(err) + } + return New(Target{BaseURL: base}, client, nil) +} + +func (h *heartbeatStub) postCount() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.posts) +} + +func heartbeatOK(id, holder string, rearmed bool) func(http.ResponseWriter) { + return func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": id, "holder": holder, "rearmed": rearmed, + "heartbeat_at": "2026-10-16T01:00:00Z", + "lease_expires_at": "2026-10-16T01:05:00Z", + }) + } +} + +func heartbeatProblem(status int, body string) func(http.ResponseWriter) { + return func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/problem+json; charset=utf-8") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + } +} + +func heartbeaterOf(t *testing.T, s *Store) issueops.Heartbeater { + t.Helper() + role, err := s.Heartbeater() + if err != nil { + t.Fatalf("Heartbeater(): %v", err) + } + return role +} + +// TestHeartbeatAgainstAServerWithoutTheTokenRefusesBeforeTheDial is the skew +// row for a new client on an old server: no issues.heartbeat in the handshake +// means a case-2 refusal NAMING the token, with nothing sent — never a +// fallback to a re-claim or a metadata stamp. +func TestHeartbeatAgainstAServerWithoutTheTokenRefusesBeforeTheDial(t *testing.T) { + stub := &heartbeatStub{} + s := stub.store(t, "issues.get", "issues.claim", "issues.release", "issues.reclaim") + + _, err := heartbeaterOf(t, s).Heartbeat(t.Context(), issueops.HeartbeatRequest{Actor: "worker", IssueID: "bd-1"}) + var capErr *wire.CapabilityError + if !errors.As(err, &capErr) { + t.Fatalf("Heartbeat on an old server = %v, want *wire.CapabilityError", err) + } + if capErr.Capability != "issues.heartbeat" { + t.Errorf("CapabilityError.Capability = %q, want issues.heartbeat", capErr.Capability) + } + var unsupported *storage.ErrUnsupported + if !errors.As(err, &unsupported) || unsupported.Capability != "issues.heartbeat" { + t.Errorf("Heartbeat on an old server = %v, want it to unwrap to *storage.ErrUnsupported{Capability: issues.heartbeat}", err) + } + if !strings.Contains(err.Error(), "issues.heartbeat") { + t.Errorf("the refusal %q does not name the token", err) + } + if n := stub.postCount(); n != 0 { + t.Errorf("%d heartbeat POSTs reached an old server, want 0", n) + } + + // The legacy front door rides the same role, so it refuses the same way. + if err := s.HeartbeatIssue(t.Context(), "bd-1", "worker"); !errors.As(err, &capErr) { + t.Errorf("HeartbeatIssue on an old server = %v, want *wire.CapabilityError", err) + } +} + +// TestHeartbeatSendsTheActorVerbatimAndProjectsTheAnswer pins the request the +// client builds (the id in the path, the actor as the one body member, sent +// untrimmed: it is the fence's subject) and the result it reads back. +func TestHeartbeatSendsTheActorVerbatimAndProjectsTheAnswer(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){heartbeatOK("bd-7", " worker ", true)}} + s := stub.store(t, "issues.heartbeat") + req := issueops.HeartbeatRequest{Actor: " worker ", IssueID: "bd-7"} + + got, err := heartbeaterOf(t, s).Heartbeat(t.Context(), req) + if err != nil { + t.Fatalf("Heartbeat: %v", err) + } + if req.Actor != " worker " || req.IssueID != "bd-7" { + t.Errorf("the request was mutated: %+v", req) + } + if stub.paths[0] != wire.PathIssues+"/bd-7"+wire.MethodHeartbeat { + t.Errorf("dialed %q, want the issue's :heartbeat", stub.paths[0]) + } + var body map[string]any + if err := json.Unmarshal([]byte(stub.posts[0]), &body); err != nil { + t.Fatalf("decode the sent body %q: %v", stub.posts[0], err) + } + if len(body) != 1 || body["actor"] != " worker " { + t.Errorf("sent body %s, want exactly {\"actor\":\" worker \"}", stub.posts[0]) + } + want := issueops.HeartbeatResult{ + IssueID: "bd-7", Holder: " worker ", Rearmed: true, + HeartbeatAt: time.Date(2026, 10, 16, 1, 0, 0, 0, time.UTC), + LeaseExpiresAt: time.Date(2026, 10, 16, 1, 5, 0, 0, time.UTC), + } + if got != want { + t.Errorf("Heartbeat = %+v, want %+v", got, want) + } +} + +// TestHeartbeatMapsTheRefusalsOntoTheRolesTypes pins each documented refusal +// to the role's typed answer. already_claimed is the sharp one: the shared +// problem table rebuilds it as the CLAIM's conflict, and on this operation it +// must read as the heartbeat's, naming the holder in the native words. +func TestHeartbeatMapsTheRefusalsOntoTheRolesTypes(t *testing.T) { + t.Run("already_claimed is a LeaseConflictError naming the holder", func(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){heartbeatProblem(http.StatusConflict, + `{"status":409,"code":"already_claimed","title":"conflict","detail":"another actor holds this issue's claim; nothing on the issue was written","assignee":"alice","issue_status":"in_progress"}`)}} + _, err := heartbeaterOf(t, stub.store(t, "issues.heartbeat")).Heartbeat(t.Context(), + issueops.HeartbeatRequest{Actor: "bob", IssueID: "bd-1"}) + var conflict *issueops.LeaseConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Heartbeat = %v (%T), want *issueops.LeaseConflictError", err, err) + } + if conflict.Holder != "alice" || conflict.IssueID != "bd-1" || conflict.Status != "in_progress" { + t.Errorf("LeaseConflictError = %+v, want Holder alice, IssueID bd-1, Status in_progress", conflict) + } + if !errors.Is(err, issueops.ErrAlreadyClaimed) || errors.Is(err, issueops.ErrLeaseNotHeld) { + t.Errorf("Heartbeat = %v: want ErrAlreadyClaimed and not ErrLeaseNotHeld", err) + } + var claim *issueops.ClaimConflictError + if errors.As(err, &claim) { + t.Errorf("Heartbeat = %v still carries the claim's ClaimConflictError", err) + } + if !strings.Contains(err.Error(), "claimed by alice") { + t.Errorf("Heartbeat error %q does not read as the native %q", err, "issue already claimed by alice") + } + }) + + t.Run("lease_not_held wraps ErrLeaseNotHeld and ErrNotClaimable", func(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){heartbeatProblem(http.StatusConflict, + `{"status":409,"code":"lease_not_held","title":"conflict","detail":"this actor holds no live claim on this issue: it is closed; nothing on the issue was written","issue_status":"closed"}`)}} + _, err := heartbeaterOf(t, stub.store(t, "issues.heartbeat")).Heartbeat(t.Context(), + issueops.HeartbeatRequest{Actor: "worker", IssueID: "bd-2"}) + var conflict *issueops.LeaseConflictError + if !errors.As(err, &conflict) || conflict.Status != "closed" || conflict.Holder != "" || conflict.Ephemeral { + t.Fatalf("Heartbeat = %v, want *issueops.LeaseConflictError{Status: closed, Holder: \"\", Ephemeral: false}", err) + } + if !errors.Is(err, issueops.ErrLeaseNotHeld) || !errors.Is(err, issueops.ErrNotClaimable) || errors.Is(err, issueops.ErrAlreadyClaimed) { + t.Errorf("Heartbeat = %v: want ErrLeaseNotHeld and ErrNotClaimable, not ErrAlreadyClaimed", err) + } + if !strings.Contains(conflict.Error(), "bd-2 status closed") || strings.Contains(err.Error(), "ephemeral") { + t.Errorf("Heartbeat error %q (typed %q): want the native %q and no mention of a wisp", err, conflict, "bd-2 status closed") + } + if !strings.Contains(err.Error(), "it is closed") { + t.Errorf("Heartbeat error %q drops the server's detail", err) + } + }) + + t.Run("lease_not_held with reason ephemeral is a wisp", func(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){heartbeatProblem(http.StatusConflict, + `{"status":409,"code":"lease_not_held","title":"conflict","detail":"this issue is an ephemeral wisp, which is never leased; nothing on the issue was written","issue_status":"in_progress","reason":"ephemeral"}`)}} + _, err := heartbeaterOf(t, stub.store(t, "issues.heartbeat")).Heartbeat(t.Context(), + issueops.HeartbeatRequest{Actor: "worker", IssueID: "bd-w"}) + var conflict *issueops.LeaseConflictError + if !errors.As(err, &conflict) || !conflict.Ephemeral || conflict.Status != "in_progress" { + t.Fatalf("Heartbeat = %v, want *issueops.LeaseConflictError{Ephemeral: true, Status: in_progress}", err) + } + if !errors.Is(err, issueops.ErrLeaseNotHeld) || !strings.Contains(conflict.Error(), "bd-w is ephemeral") { + t.Errorf("Heartbeat error %q (typed %q): want ErrLeaseNotHeld in the native %q", err, conflict, "bd-w is ephemeral") + } + }) + + t.Run("not_found is ErrNotFound", func(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){heartbeatProblem(http.StatusNotFound, + `{"status":404,"code":"not_found","title":"not found"}`)}} + _, err := heartbeaterOf(t, stub.store(t, "issues.heartbeat")).Heartbeat(t.Context(), + issueops.HeartbeatRequest{Actor: "worker", IssueID: "bd-404"}) + if !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Heartbeat = %v, want storage.ErrNotFound", err) + } + }) + + t.Run("an invalid actor is the role's field error", func(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){heartbeatProblem(http.StatusBadRequest, + `{"status":400,"code":"invalid_argument","title":"bad","param":"actor","reason":"invalid_value","detail":"actor must not contain control characters"}`)}} + _, err := heartbeaterOf(t, stub.store(t, "issues.heartbeat")).Heartbeat(t.Context(), + issueops.HeartbeatRequest{Actor: "work\u0007er", IssueID: "bd-3"}) + var field *issueops.HeartbeatFieldError + if !errors.As(err, &field) || field.Field != issueops.HeartbeatFieldActor || !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Heartbeat = %v, want *issueops.HeartbeatFieldError{Field: actor} matching ErrValidation", err) + } + }) +} + +// TestHeartbeatReplaysABusyAnswer pins the replayable flag: the one write this +// client marks idempotent, because a repeat only moves the expiry later. A 503 +// is retried, not handed to a worker that stops on any nonzero exit. +func TestHeartbeatReplaysABusyAnswer(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){ + func(w http.ResponseWriter) { + w.Header().Set("Retry-After", "0") + heartbeatProblem(http.StatusServiceUnavailable, `{"status":503,"code":"busy","title":"busy"}`)(w) + }, + heartbeatOK("bd-9", "worker", false), + }} + got, err := heartbeaterOf(t, stub.store(t, "issues.heartbeat")).Heartbeat(t.Context(), + issueops.HeartbeatRequest{Actor: "worker", IssueID: "bd-9"}) + if err != nil { + t.Fatalf("Heartbeat after one busy answer = %v, want the replay's success", err) + } + if got.IssueID != "bd-9" { + t.Errorf("Heartbeat = %+v, want bd-9", got) + } + if n := stub.postCount(); n != 2 { + t.Errorf("%d heartbeat POSTs, want 2 (the busy one and its replay)", n) + } +} + +// TestHeartbeatRefusesBeforeTheDial pins the request rules: each is the role's +// *HeartbeatFieldError, and none reaches the server. +func TestHeartbeatRefusesBeforeTheDial(t *testing.T) { + stub := &heartbeatStub{} + role := heartbeaterOf(t, stub.store(t, "issues.heartbeat")) + for name, tc := range map[string]struct { + req issueops.HeartbeatRequest + field string + }{ + "blank actor": {issueops.HeartbeatRequest{Actor: " \t", IssueID: "bd-1"}, issueops.HeartbeatFieldActor}, + "over-long actor": {issueops.HeartbeatRequest{Actor: strings.Repeat("a", 256), IssueID: "bd-1"}, issueops.HeartbeatFieldActor}, + "empty id": {issueops.HeartbeatRequest{Actor: "worker"}, issueops.HeartbeatFieldIssueID}, + } { + t.Run(name, func(t *testing.T) { + _, err := role.Heartbeat(t.Context(), tc.req) + var field *issueops.HeartbeatFieldError + if !errors.As(err, &field) || field.Field != tc.field || !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("Heartbeat(%s) = %v, want *issueops.HeartbeatFieldError{Field: %s}", name, err, tc.field) + } + }) + } + if n := stub.postCount(); n != 0 { + t.Errorf("%d invalid heartbeats reached the server, want 0", n) + } +} + +// TestHeartbeatValidationMatchesTheSharedValidator binds heartbeater.go's +// restated rules to internal/storage/issueops.ValidateHeartbeatRequest, for +// TestReclaimValidationMatchesTheSharedValidator's reason. +func TestHeartbeatValidationMatchesTheSharedValidator(t *testing.T) { + for name, req := range map[string]issueops.HeartbeatRequest{ + "valid": {Actor: "worker", IssueID: "bd-1"}, + "padded actor is valid": {Actor: " worker ", IssueID: "bd-1"}, + "blank actor": {Actor: " ", IssueID: "bd-1"}, + "absent actor": {IssueID: "bd-1"}, + "at the column": {Actor: strings.Repeat("a", 255), IssueID: "bd-1"}, + "over the column": {Actor: strings.Repeat("a", 256), IssueID: "bd-1"}, + "empty id": {Actor: "worker"}, + "blank actor and id": {}, + "padded id is an id": {Actor: "worker", IssueID: " bd-1 "}, + } { + t.Run(name, func(t *testing.T) { + mine := validateHeartbeatRequest(req) + shared := storeops.ValidateHeartbeatRequest(req) + if (mine == nil) != (shared == nil) { + t.Fatalf("validateHeartbeatRequest = %v, storeops.ValidateHeartbeatRequest = %v: the client's restatement has drifted", mine, shared) + } + if mine == nil { + return + } + var mineField, sharedField *issueops.HeartbeatFieldError + if !errors.As(mine, &mineField) || !errors.As(shared, &sharedField) { + t.Fatalf("validateHeartbeatRequest = %v, shared = %v: want both *issueops.HeartbeatFieldError", mine, shared) + } + if *mineField != *sharedField { + t.Errorf("validateHeartbeatRequest = %+v, shared = %+v", *mineField, *sharedField) + } + }) + } +} + +// TestLegacyHeartbeatIssueRidesTheRole pins the raw front door onto the role: +// one POST with the same body, and the role's typed refusal back. +func TestLegacyHeartbeatIssueRidesTheRole(t *testing.T) { + stub := &heartbeatStub{replies: []func(http.ResponseWriter){ + heartbeatOK("bd-5", "worker", false), + heartbeatProblem(http.StatusConflict, + `{"status":409,"code":"already_claimed","title":"conflict","assignee":"alice","issue_status":"in_progress"}`), + }} + s := stub.store(t, "issues.heartbeat") + if err := s.HeartbeatIssue(t.Context(), "bd-5", "worker"); err != nil { + t.Fatalf("HeartbeatIssue: %v", err) + } + err := s.HeartbeatIssue(t.Context(), "bd-5", "worker") + var conflict *issueops.LeaseConflictError + if !errors.As(err, &conflict) || conflict.Holder != "alice" || !errors.Is(err, storage.ErrAlreadyClaimed) { + t.Errorf("HeartbeatIssue against a foreign holder = %v, want *issueops.LeaseConflictError{Holder: alice}", err) + } + if err := s.HeartbeatIssue(t.Context(), "bd-5", ""); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("HeartbeatIssue with no actor = %v, want ErrValidation before the dial", err) + } + if n := stub.postCount(); n != 2 { + t.Errorf("%d heartbeat POSTs, want 2", n) + } +} diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go index 4ca99dc27..d89349ae2 100644 --- a/internal/httpclient/served_harness_test.go +++ b/internal/httpclient/served_harness_test.go @@ -303,6 +303,8 @@ func serveConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { fail("BatchGetter") cfg.LeaseReclaimer, err = s.LeaseReclaimer() fail("LeaseReclaimer") + cfg.Heartbeater, err = s.Heartbeater() + fail("Heartbeater") // THE JOURNAL IS NOT AN ACCESSOR, alone among the roles above: it is reached // by TYPE ASSERTION, because it is not on storage.DoltStorage and a backend diff --git a/internal/httpclient/served_heartbeater_test.go b/internal/httpclient/served_heartbeater_test.go new file mode 100644 index 000000000..268779586 --- /dev/null +++ b/internal/httpclient/served_heartbeater_test.go @@ -0,0 +1,147 @@ +//go:build cgo + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// The Heartbeater contract, run through the http client's accessor against a +// real bd serve: POST /v0/beads/issues/{id}:heartbeat. +// Nothing is parked: the id rides in the path and the actor is the body's one +// member, and every request rule the role lists is raised before the dial +// (heartbeater.go). +// +// Seeding and every precondition (the claim, the close, the release, the +// reclaim, the status move) go through the REFERENCE store's own roles, never +// the wire, per the harness's binding rule; the post-state is read raw. +// +// THE HOOK CASE runs the heartbeater off a HookFiringStore wrapped around the +// client, the chain cmd/bd composes for a connected workspace: bd serve runs +// no hooks, so it is the CLIENT's decorator that must fire none for a +// refresh — the native refresh fires none on any route (a re-arm, recorded as +// a mutation, fires on_update). +func newServedHeartbeaterFixture(t *testing.T, prefix string) conformance.HeartbeaterFixture { + t.Helper() + env := newServedEnv(t, prefix) + ref := env.reference + heartbeater, err := env.subject.Heartbeater() + if err != nil { + t.Fatalf("Heartbeater(): %v", err) + } + claimer, err := ref.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := ref.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + releaser, err := ref.Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + reclaimer, err := ref.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := heartbeater + if runner != nil { + if hooked, err = storage.NewHookFiringStore(env.subject, runner).Heartbeater(); err != nil { + t.Fatalf("hooked Heartbeater(): %v", err) + } + } + return conformance.HeartbeaterFixture{ + IssuePrefix: env.prefix, + Heartbeater: heartbeater, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + Claimer: claimer, + Lifecycle: lifecycle, + Releaser: releaser, + LeaseReclaimer: reclaimer, + Exec: env.exec, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + SetJournalEnabled: ref.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: ref.SetVersionedHistoryEnabled, + HookedHeartbeater: hooked, + FiredUpdates: fired, + } +} + +func TestServedHeartbeaterRefreshesTheHoldersLease(t *testing.T) { + conformance.RunHeartbeaterRefreshesTheHoldersLease(t, t.Context(), newServedHeartbeaterFixture(t, "hhb00")) +} + +func TestServedHeartbeaterChangesNothingOnTheRow(t *testing.T) { + conformance.RunHeartbeaterChangesNothingOnTheRow(t, t.Context(), newServedHeartbeaterFixture(t, "hhb01")) +} + +func TestServedHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins(t *testing.T) { + conformance.RunHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins(t, t.Context(), newServedHeartbeaterFixture(t, "hhb02")) +} + +func TestServedHeartbeaterRefusesAForeignHolder(t *testing.T) { + conformance.RunHeartbeaterRefusesAForeignHolder(t, t.Context(), newServedHeartbeaterFixture(t, "hhb03")) +} + +func TestServedHeartbeaterRefusesAfterClose(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterClose(t, t.Context(), newServedHeartbeaterFixture(t, "hhb04")) +} + +func TestServedHeartbeaterRefusesAfterRelease(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterRelease(t, t.Context(), newServedHeartbeaterFixture(t, "hhb05")) +} + +func TestServedHeartbeaterRefusesAfterReclaim(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterReclaim(t, t.Context(), newServedHeartbeaterFixture(t, "hhb06")) +} + +func TestServedHeartbeaterRefusesAfterStatusMovedOffInProgress(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterStatusMovedOffInProgress(t, t.Context(), newServedHeartbeaterFixture(t, "hhb07")) +} + +func TestServedHeartbeaterRefusesAWisp(t *testing.T) { + conformance.RunHeartbeaterRefusesAWisp(t, t.Context(), newServedHeartbeaterFixture(t, "hhb08")) +} + +func TestServedHeartbeaterRearmsAHandDoledClaim(t *testing.T) { + conformance.RunHeartbeaterRearmsAHandDoledClaim(t, t.Context(), newServedHeartbeaterFixture(t, "hhb09")) +} + +func TestServedHeartbeaterAnswersNotFoundForAnAbsentID(t *testing.T) { + conformance.RunHeartbeaterAnswersNotFoundForAnAbsentID(t, t.Context(), newServedHeartbeaterFixture(t, "hhb10")) +} + +func TestServedHeartbeaterRefusesABlankOrOverLongActor(t *testing.T) { + conformance.RunHeartbeaterRefusesABlankOrOverLongActor(t, t.Context(), newServedHeartbeaterFixture(t, "hhb11")) +} + +func TestServedHeartbeaterDoesNotMutateTheRequest(t *testing.T) { + conformance.RunHeartbeaterDoesNotMutateTheRequest(t, t.Context(), newServedHeartbeaterFixture(t, "hhb12")) +} + +func TestServedHeartbeaterASameSecondHeartbeatIsNotARearm(t *testing.T) { + conformance.RunHeartbeaterASameSecondHeartbeatIsNotARearm(t, t.Context(), newServedHeartbeaterFixture(t, "hhb13")) +} + +func TestServedHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue(t, t.Context(), newServedHeartbeaterFixture(t, "hhb14")) +} + +func TestServedHeartbeaterRefusesAnOrphanedLeaseOnAClosedIssue(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAClosedIssue(t, t.Context(), newServedHeartbeaterFixture(t, "hhb15")) +} + +func TestServedHeartbeaterRefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds(t, t.Context(), newServedHeartbeaterFixture(t, "hhb16")) +} + +func TestServedHeartbeaterCommitsOnlyItsOwnTables(t *testing.T) { + conformance.RunHeartbeaterCommitsOnlyItsOwnTables(t, t.Context(), newServedHeartbeaterFixture(t, "hhb17")) +} diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index b94c31275..9d3bf3a19 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -243,6 +243,9 @@ func serveRoles(store storage.DoltStorage) (httpapi.Config, error) { if cfg.LeaseReclaimer, err = store.LeaseReclaimer(); err != nil { return cfg, err } + if cfg.Heartbeater, err = store.Heartbeater(); err != nil { + return cfg, err + } return cfg, nil } diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index c92f73d69..fd47b5b54 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -402,11 +402,6 @@ func (unsupportedDoltStorage) HasRemote(_ context.Context, _ string) (_ bool, er return } -func (unsupportedDoltStorage) HeartbeatIssue(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("HeartbeatIssue") - return -} - func (unsupportedDoltStorage) History(_ context.Context, _ string) (_ []*storage.HistoryEntry, err error) { err = errUnsupported("History") return @@ -727,4 +722,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (142 of 203 methods generated; 61 left to this package's hand-written set). +// NOTE: partial shell (141 of 204 methods generated; 63 left to this package's hand-written set). diff --git a/internal/httpclient/wire/client.go b/internal/httpclient/wire/client.go index 7f2a82b22..7c956bb96 100644 --- a/internal/httpclient/wire/client.go +++ b/internal/httpclient/wire/client.go @@ -112,10 +112,10 @@ type Options struct { // arrived is never cut off by it. Such a request that hits it, or whose // connection is reset, is retried once on another connection, with no // header timer and only what is left of the overall Timeout, counted from - // when the first attempt was sent. A write is never - // retried and is bounded by the overall Timeout alone (bd serve answers a - // write after it commits). A written request whose headers do not come - // back in time fails with *ResponseHeaderTimeoutError. + // when the first attempt was sent. A write that is not Request.Idempotent + // is never retried and is bounded by the overall Timeout alone (bd serve + // answers a write after it commits). A written request whose headers do + // not come back in time fails with *ResponseHeaderTimeoutError. ResponseHeaderTimeout time.Duration // OnStallRetry, when set, is told about every such retry. The process-wide // count is StallRetryCount either way. @@ -276,7 +276,9 @@ type Request struct { Actor string // Idempotent marks a read carried on a non-GET method (a POST batch get) // as safe to replay, the way a GET is (see Replayable). Never set it on a - // write. + // write whose replay could double-apply. The one write that sets it is the + // lease heartbeat (heartbeatIssue): a repeat only moves the expiry later, + // and a replay after the lease is gone fails the holder fence. Idempotent bool // Timeout, when positive, RAISES the client's per-request ceiling for this // one request — never lowers it, and never bounds a client whose diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 6dbc86efd..5eae5a69c 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -107,6 +107,10 @@ var opCapability = map[string]string{ // with a typed capability error instead of dialing a path the older server // would answer as a claim of an issue called ":reclaim". OpReclaimIssues: "issues.reclaim", + // The lease refresh, dialed by httpHeartbeater. A server that predates it + // does not advertise the token, so Preflight refuses locally with a typed + // capability error rather than dialing a route the older server lacks. + OpHeartbeatIssue: "issues.heartbeat", // The advance-a-molecule role (molecule.go) dials it when served and // refuses before dialing when not; it never composes a client-side copy. OpAdvanceMolecule: "issues.advanceMolecule", diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 08861e232..c31a5809d 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -59,6 +59,7 @@ const ( OpForgetMemory = "forgetMemory" OpBatchGetIssues = "batchGetIssues" OpReclaimIssues = "reclaimIssues" + OpHeartbeatIssue = "heartbeatIssue" ) // The paths that carry no caller-supplied segment. A path with one is built by @@ -105,6 +106,7 @@ const ( MethodAdvanceMolecule = ":advanceMolecule" MethodRelease = ":release" MethodCASMetadata = ":casMetadata" + MethodHeartbeat = ":heartbeat" ) // The sub-resource collections that hang off the issue-detail path. Unlike a @@ -130,7 +132,7 @@ func IssuePath(id string) (string, error) { // final segment: /v0/beads/issues/{id}:claim and its two siblings. func IssueMethodPath(id, method string) (string, error) { switch method { - case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata, MethodAdvanceMolecule: + case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata, MethodAdvanceMolecule, MethodHeartbeat: default: return "", fmt.Errorf("no custom method %q on the issue resource", method) } diff --git a/internal/httpclient/wire/paths_test.go b/internal/httpclient/wire/paths_test.go index abeddc981..96a9e0d4b 100644 --- a/internal/httpclient/wire/paths_test.go +++ b/internal/httpclient/wire/paths_test.go @@ -133,6 +133,7 @@ func TestIssueMethodPathBuildsTheSuffixIntoTheFinalSegment(t *testing.T) { {MethodClose, "/v0/beads/issues/ga-1:close"}, {MethodReopen, "/v0/beads/issues/ga-1:reopen"}, {MethodRelease, "/v0/beads/issues/ga-1:release"}, + {MethodHeartbeat, "/v0/beads/issues/ga-1:heartbeat"}, } for _, tc := range cases { got, err := IssueMethodPath("ga-1", tc.method) diff --git a/internal/httpclient/wire/problem.go b/internal/httpclient/wire/problem.go index 106cfea73..bd42f35c7 100644 --- a/internal/httpclient/wire/problem.go +++ b/internal/httpclient/wire/problem.go @@ -299,6 +299,11 @@ const ReasonProjectMismatch = "project_mismatch" // tells this refusal apart from an ordinary malformed argument. const ReasonWireRevisionUnsupported = "wire_revision_unsupported" +// ReasonEphemeral is the Problem `reason` that qualifies a lease_not_held: the +// id names a wisp, which is never leased. Spelled exactly as the server's +// httpapi.ReasonEphemeral (held to it by the vocabulary test). +const ReasonEphemeral = "ephemeral" + // ErrWireRevisionUnsupported reports that bd serve refused this client's // declared Bd-Wire-Revision header as older than the revision it now // requires. See WireRevisionUnsupportedError. @@ -633,6 +638,31 @@ var codeSentinel = map[string]func(*ProblemError, target) error{ // and the one whose message does not assert a fact the wire never sent. "not_releasable": func(*ProblemError, target) error { return issueops.ErrNotReleasable }, + // lease_not_held is the HEARTBEAT's refusal: the actor holds no live claim + // (open, closed, released, reclaimed, moved off in_progress, or a wisp). + // httpclient's Heartbeater reaches it (heartbeater.go). The typed error is + // rebuilt whole from `issue_status` and `reason: ephemeral` (a wisp), and + // it wraps ErrLeaseNotHeld, which also matches ErrNotClaimable, in the + // native heartbeat's own words (" status ", " is + // ephemeral"). + "lease_not_held": func(e *ProblemError, t target) error { + id := refusedIssueID(e, t) + status := deref(e.IssueStatus) + conflict := &issueops.LeaseConflictError{ + IssueID: id, + Status: issueops.Status(status), + Ephemeral: e.Reason == ReasonEphemeral, + Err: issueops.ErrLeaseNotHeld, + } + switch { + case conflict.Ephemeral: + conflict.Err = fmt.Errorf("%w: %s is ephemeral", issueops.ErrLeaseNotHeld, id) + case status != "": + conflict.Err = fmt.Errorf("%w: %s status %s", issueops.ErrLeaseNotHeld, id, status) + } + return conflict + }, + // [Added: OSS publishes notes_overwrite_refused (internal/httpapi/problem.go // CodeNotesOverwrite, status 409) for an update whose Patch.Notes would // replace existing notes without an explicit overwrite flag. This code had diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index ec690e114..7eeea64fc 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -110,8 +110,8 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) } // The dimension is pinned so the matrix cannot shrink silently: the first - // slice served six operations and the tip surface has forty-four, so - // thirty-eight of them are things a first-slice server has never heard of. The + // slice served six operations and the tip surface has forty-five, so + // thirty-nine of them are things a first-slice server has never heard of. The // wire wave moved this from twenty-seven to thirty — claimNext, release and // count — wave 2 to thirty-two, adding the dependency-edge count and the // related read, the mini-sync to thirty-five: the comment write and the two @@ -119,9 +119,10 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat // stale-lease sweep to thirty-seven. Each is // an operation a first-slice server never served and this client does not // yet dial, which is exactly the shape the refusal matrix is about. The - // molecule advance (S16) makes thirty-eight. - if len(refused) != 38 { - t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 38", len(refused)) + // molecule advance (S16) makes thirty-eight, and the lease refresh + // (issues.heartbeat) thirty-nine. + if len(refused) != 39 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 39", len(refused)) } // The compat op forces the handshake yet preflights clean: issues.claim is on diff --git a/internal/httpclient/wire/stall_retry.go b/internal/httpclient/wire/stall_retry.go index c0f3a90dc..e8fc09f85 100644 --- a/internal/httpclient/wire/stall_retry.go +++ b/internal/httpclient/wire/stall_retry.go @@ -35,9 +35,9 @@ import ( // slow body that keeps streaming is governed by DefaultTimeout (and the // caller's context) alone. // -// The timer applies to replayable requests (see Request.Replayable). A write is -// bounded by the client's overall timeout instead, because bd serve answers a -// write only after it commits; see roundTrip. +// The timer applies to replayable requests (see Request.Replayable). Any other +// write is bounded by the client's overall timeout instead, because bd serve +// answers a write only after it commits; see roundTrip. const DefaultResponseHeaderTimeout = 15 * time.Second // ErrResponseHeaderTimeout is the sentinel a *ResponseHeaderTimeoutError diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 3c1353c6a..b3144310f 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -69,6 +69,7 @@ func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { {OpWatchEvents, httpapi.OpWatchEvents}, {OpBatchGetIssues, httpapi.OpBatchGetIssues}, {OpReclaimIssues, httpapi.OpReclaimIssues}, + {OpHeartbeatIssue, httpapi.OpHeartbeatIssue}, {OpAdvanceMolecule, httpapi.OpAdvanceMolecule}, } @@ -187,6 +188,9 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if ReasonProjectMismatch != string(httpapi.ReasonProjectMismatch) { t.Errorf("ReasonProjectMismatch = %q, server says %q", ReasonProjectMismatch, httpapi.ReasonProjectMismatch) } + if ReasonEphemeral != string(httpapi.ReasonEphemeral) { + t.Errorf("ReasonEphemeral = %q, server says %q", ReasonEphemeral, httpapi.ReasonEphemeral) + } if WireRevisionHeader != httpapi.WireRevisionHeader { t.Errorf("WireRevisionHeader = %q, server says %q", WireRevisionHeader, httpapi.WireRevisionHeader) } diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 7252f4b68..1092cb69b 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "hook_dependency_editor.go", "hook_edgereader.go", "hook_graph_counter.go", + "hook_heartbeater.go", "hook_init_verifier.go", "hook_issue_claimer.go", "hook_issue_operations.go", @@ -92,6 +93,7 @@ go_test( "hook_decorator_internal_test.go", "hook_decorator_test.go", "hook_dependency_editor_test.go", + "hook_heartbeater_test.go", "hook_issue_claimer_test.go", "hook_issue_operations_test.go", "hook_issue_reader_test.go", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index f81b80beb..6ce9c4028 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -34,6 +34,7 @@ go_library( "filters.go", "gateway_credential.go", "graph_counter.go", + "heartbeater.go", "history.go", "init_verifier.go", "issue_claimer.go", @@ -237,6 +238,7 @@ go_test( "git_remote_test.go", "gm2g3g5r_leak_repro_test.go", "graph_counter_contract_test.go", + "heartbeater_contract_test.go", "history_presence_contract_test.go", "history_test.go", "identity_mismatch_message_test.go", diff --git a/internal/storage/dolt/heartbeater.go b/internal/storage/dolt/heartbeater.go new file mode 100644 index 000000000..2233cc8f2 --- /dev/null +++ b/internal/storage/dolt/heartbeater.go @@ -0,0 +1,59 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// Heartbeater returns the lease-refresh surface for this store. +func (s *DoltStore) Heartbeater() (issueops.Heartbeater, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "Heartbeater", Backend: "nil"} + } + return &heartbeater{store: s}, nil +} + +// heartbeater answers a Heartbeat from one retrying transaction around the +// shared body, storeops.ExecuteHeartbeatInTx, through the issue-operation +// funnel. A REFRESH stages nothing (the leases table is dolt_ignored), so it +// mints no commit and no history (bd-lrgn1); a RE-ARM is recorded as a +// mutation and gets the version-control commit LeaseRearmVersionCommit names. +// The retry replays a heartbeat that loses Dolt's optimistic merge to a +// concurrent reclaim, release or close. +type heartbeater struct{ store *DoltStore } + +var _ issueops.Heartbeater = (*heartbeater)(nil) + +func (h *heartbeater) Heartbeat(ctx context.Context, request issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + if err := storeops.ValidateHeartbeatRequest(request); err != nil { + return issueops.HeartbeatResult{}, err + } + var result issueops.HeartbeatResult + var refusal error + if err := h.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + result, err = storeops.ExecuteHeartbeatInTx(ctx, tx, request) + // A lease refusal commits (it may have deleted an orphaned lease + // row) and is reported after the commit; see HeartbeatRefusalCommits. + refusal = nil + if storeops.HeartbeatRefusalCommits(err) { + refusal = err + return nil, "", nil + } + if err != nil { + return nil, "", err + } + tables, msg := storeops.LeaseRearmVersionCommit(request.IssueID, result.Rearmed) + return tables, msg, nil + }); err != nil { + return issueops.HeartbeatResult{}, err + } + if refusal != nil { + return issueops.HeartbeatResult{}, refusal + } + return result, nil +} diff --git a/internal/storage/dolt/heartbeater_contract_test.go b/internal/storage/dolt/heartbeater_contract_test.go new file mode 100644 index 000000000..0113aaaf8 --- /dev/null +++ b/internal/storage/dolt/heartbeater_contract_test.go @@ -0,0 +1,142 @@ +package dolt + +import ( + "context" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// TestHeartbeaterContract runs the Heartbeater contract against the +// server-backed store, which wraps internal/storage/issueops.ExecuteHeartbeatInTx +// in a retrying transaction that makes no Dolt commit for a refresh and one +// for a re-arm. +func TestHeartbeaterContract(t *testing.T) { + fixture, ctx, cleanup := newDoltHeartbeaterFixture(t, "hbc") + defer cleanup() + t.Run("RefreshesTheHoldersLease", func(t *testing.T) { + conformance.RunHeartbeaterRefreshesTheHoldersLease(t, ctx, fixture) + }) + t.Run("ChangesNothingOnTheRow", func(t *testing.T) { + conformance.RunHeartbeaterChangesNothingOnTheRow(t, ctx, fixture) + }) + t.Run("AGuardedWriteWithThePreHeartbeatRevisionStillWins", func(t *testing.T) { + conformance.RunHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins(t, ctx, fixture) + }) + t.Run("RefusesAForeignHolder", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAForeignHolder(t, ctx, fixture) + }) + t.Run("RefusesAfterClose", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterClose(t, ctx, fixture) + }) + t.Run("RefusesAfterRelease", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterRelease(t, ctx, fixture) + }) + t.Run("RefusesAfterReclaim", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterReclaim(t, ctx, fixture) + }) + t.Run("RefusesAfterStatusMovedOffInProgress", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterStatusMovedOffInProgress(t, ctx, fixture) + }) + t.Run("RefusesAWisp", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAWisp(t, ctx, fixture) + }) + t.Run("RearmsAHandDoledClaim", func(t *testing.T) { + conformance.RunHeartbeaterRearmsAHandDoledClaim(t, ctx, fixture) + }) + t.Run("AnswersNotFoundForAnAbsentID", func(t *testing.T) { + conformance.RunHeartbeaterAnswersNotFoundForAnAbsentID(t, ctx, fixture) + }) + t.Run("RefusesABlankOrOverLongActor", func(t *testing.T) { + conformance.RunHeartbeaterRefusesABlankOrOverLongActor(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheRequest", func(t *testing.T) { + conformance.RunHeartbeaterDoesNotMutateTheRequest(t, ctx, fixture) + }) + t.Run("CommitsOnlyItsOwnTables", func(t *testing.T) { + conformance.RunHeartbeaterCommitsOnlyItsOwnTables(t, ctx, fixture) + }) + t.Run("ASameSecondHeartbeatIsNotARearm", func(t *testing.T) { + conformance.RunHeartbeaterASameSecondHeartbeatIsNotARearm(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAReleasedIssue", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAClosedIssue", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAClosedIssue(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds(t, ctx, fixture) + }) +} + +// newDoltHeartbeaterFixture composes the frozen role kit with this backend's +// accessors, plus the heartbeater taken off a HookFiringStore so the quiet case +// observes the decorator a command actually holds. +func newDoltHeartbeaterFixture(t *testing.T, prefix string) (conformance.HeartbeaterFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + stop := func() { + cancel() + storeCleanup() + } + fail := func(what string, err error) { + stop() + t.Fatalf("%s: %v", what, err) + } + heartbeater, err := store.Heartbeater() + if err != nil { + fail("Heartbeater()", err) + } + claimer, err := store.IssueClaimer() + if err != nil { + fail("IssueClaimer()", err) + } + lifecycle, err := store.IssueLifecycle() + if err != nil { + fail("IssueLifecycle()", err) + } + releaser, err := store.Releaser() + if err != nil { + fail("Releaser()", err) + } + reclaimer, err := store.LeaseReclaimer() + if err != nil { + fail("LeaseReclaimer()", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + var hooked = heartbeater + if runner != nil { + if hooked, err = storage.NewHookFiringStore(store, runner).Heartbeater(); err != nil { + fail("hooked Heartbeater()", err) + } + } + kit := newDoltRoleFixtureKit(store, prefix) + return conformance.HeartbeaterFixture{ + IssuePrefix: kit.IssuePrefix, + Heartbeater: heartbeater, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + Claimer: claimer, + Lifecycle: lifecycle, + Releaser: releaser, + LeaseReclaimer: reclaimer, + QueryScalar: kit.QueryScalar, + CountHistory: kit.CountHistory, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + for _, stmt := range statements { + if _, err := store.db.ExecContext(ctx, stmt.Query, stmt.Args...); err != nil { + return fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return nil + }, + SetJournalEnabled: store.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: store.SetVersionedHistoryEnabled, + HookedHeartbeater: hooked, + FiredUpdates: fired, + }, ctx, stop +} diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index b6a8362b6..0fe3d4cf8 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -445,20 +445,41 @@ func (s *DoltStore) verifiedReadyClaim(ctx context.Context, actor string, write // HeartbeatIssue refreshes the lease on an issue actor holds in_progress, // pushing lease_expires_at forward on its row in the ephemeral leases table -// (see issueops.lease). Deliberately NO DOLT_ADD/DOLT_COMMIT: the leases -// table is dolt_ignored, so a heartbeat mints no commit and no history — this -// is the whole point of bd-lrgn1 (fleet heartbeats were the dominant source -// of unbounded reachable history). Wrapped in withRetryTx so a heartbeat that -// loses Dolt's optimistic merge to a concurrent reclaim/close on the same -// lease row is replayed against a fresh snapshot rather than surfaced. +// (see issueops.lease). A REFRESH stages nothing: the leases table is +// dolt_ignored, so it mints no Dolt commit and no history — the whole point of +// bd-lrgn1 (fleet heartbeats were the dominant source of unbounded reachable +// history). A RE-ARM (a live claim with no lease row the caller holds) is +// recorded as a mutation and gets the one version-control commit +// issueops.LeaseRearmVersionCommit names. Wrapped in withRetryTx so a +// heartbeat that loses Dolt's optimistic merge to a concurrent reclaim/close +// on the same lease row is replayed against a fresh snapshot rather than +// surfaced. func (s *DoltStore) HeartbeatIssue(ctx context.Context, id, actor string) error { if s.isActiveWisp(ctx, id) { // Wisps are ephemeral and never leased; nothing to heartbeat. return fmt.Errorf("%w: %s is ephemeral", storage.ErrNotClaimable, id) } - return s.withRetryTx(ctx, func(tx *sql.Tx) error { - return issueops.HeartbeatIssueInTx(ctx, tx, id, actor) - }) + // A refusal that read the issue row COMMITS (it may have deleted the + // caller's orphaned lease row) and is returned after; one whose issue read + // failed is returned from the callback, so its transaction rolls back + // (issueops.LeaseRenewalRefusalCommits). + var refusal error + if err := s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (issueops.ChangedTables, string, error) { + renewal, err := issueops.RenewLeaseInTx(ctx, tx, id, actor) + refusal = nil + if issueops.LeaseRenewalRefusalCommits(err) { + refusal = err + return nil, "", nil + } + if err != nil { + return nil, "", err + } + tables, msg := issueops.LeaseRearmVersionCommit(id, renewal.Rearmed) + return tables, msg, nil + }); err != nil { + return err + } + return refusal } // ReclaimExpiredLeases reverts in_progress issues whose lease expired more than diff --git a/internal/storage/dolt/lease_test.go b/internal/storage/dolt/lease_test.go index 02ef454d9..386efa04d 100644 --- a/internal/storage/dolt/lease_test.go +++ b/internal/storage/dolt/lease_test.go @@ -848,6 +848,106 @@ func TestHeartbeatRevivesExpiredLease(t *testing.T) { } } +// TestHeartbeatIssueCommitsARefusalOnlyWhenItReadTheIssue pins the legacy +// door's commit rule for a refusal (issueops.LeaseRenewalRefusalCommits, the +// role's HeartbeatRefusalCommits rule). A refusal that read the issue row +// commits, so the orphaned lease row it deletes stays deleted. A refusal whose +// issue read failed rolls back: a lease row with no issue row takes the fast +// path's UPDATE, which extends its expiry, then fails the live-claim check, and +// committing that would keep the extension while answering an error. It also +// pins the delete's holder fence: the caller's orphan held under another +// spelling of its identity is deleted, another identity's orphan survives. +func TestHeartbeatIssueCommitsARefusalOnlyWhenItReadTheIssue(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + plantLease := func(id, holder string) { + t.Helper() + granted := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at, granted_node) + VALUES (?, ?, ?, ?, ?, '') + `, id, holder, granted, granted.Add(time.Minute), granted); err != nil { + t.Fatalf("plant a lease row on %s: %v", id, err) + } + } + seedOpenIssue := func(id string) { + t.Helper() + if err := store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "lease " + id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + leaseRows := func(id string) int { + t.Helper() + var rows int + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM leases WHERE issue_id = ?`, id).Scan(&rows); err != nil { + t.Fatalf("count lease rows on %s: %v", id, err) + } + return rows + } + hbCtx := issueops.WithLeaseTTL(ctx, time.Hour) + + // No issue row: the refusal read nothing, so the fast path's extension + // must roll back. + plantLease("lease-noissue", "alice") + var before string + if err := store.db.QueryRowContext(ctx, + `SELECT CAST(lease_expires_at AS CHAR) FROM leases WHERE issue_id = ?`, "lease-noissue", + ).Scan(&before); err != nil { + t.Fatalf("read planted expiry: %v", err) + } + if err := store.HeartbeatIssue(hbCtx, "lease-noissue", "alice"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("heartbeat on a lease row with no issue row err = %v, want ErrNotClaimable", err) + } + var after string + if err := store.db.QueryRowContext(ctx, + `SELECT CAST(lease_expires_at AS CHAR) FROM leases WHERE issue_id = ?`, "lease-noissue", + ).Scan(&after); err != nil { + t.Fatalf("read expiry after the refusal: %v", err) + } + if after != before { + t.Errorf("refused heartbeat committed the fast path's extension: lease_expires_at %s -> %s, want unchanged", before, after) + } + + // An open issue: the refusal read it and deleted the caller's orphaned + // row, and that delete must commit. + seedOpenIssue("lease-orphan") + plantLease("lease-orphan", "alice") + if err := store.HeartbeatIssue(hbCtx, "lease-orphan", "alice"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("heartbeat over an orphaned lease on an open issue err = %v, want ErrNotClaimable", err) + } + if rows := leaseRows("lease-orphan"); rows != 0 { + t.Errorf("lease rows on lease-orphan after the refusal = %d, want 0: the refusal's orphan delete must commit", rows) + } + + // The caller's orphan held under another spelling of its identity + // (ga-v2k49): the delete's holder fence is judged under actorMatches, as + // the in-place refresh's is, so the row is the caller's and is deleted. + seedOpenIssue("lease-orphan-spelling") + plantLease("lease-orphan-spelling", "gastown.mayor") + if err := store.HeartbeatIssue(hbCtx, "lease-orphan-spelling", "gastown__mayor"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("cross-spelling heartbeat over an orphaned lease err = %v, want ErrNotClaimable", err) + } + if rows := leaseRows("lease-orphan-spelling"); rows != 0 { + t.Errorf("lease rows on lease-orphan-spelling after a cross-spelling refusal = %d, want 0: the orphan is the caller's under another spelling", rows) + } + + // Control: a genuinely different identity's orphan is not the caller's, + // so the refusal leaves it alone. + seedOpenIssue("lease-orphan-other") + plantLease("lease-orphan-other", "gastown.dog-3") + if err := store.HeartbeatIssue(hbCtx, "lease-orphan-other", "gastown__mayor"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("heartbeat over another identity's orphaned lease err = %v, want ErrNotClaimable", err) + } + if rows := leaseRows("lease-orphan-other"); rows != 1 { + t.Errorf("lease rows on lease-orphan-other after the refusal = %d, want 1: a refusal deletes only the caller's own row", rows) + } +} + // TestConcurrentReapersAreBenign pins L5.5: two reapers racing the same stale // set neither error nor double-report a reclaim, and do not disturb live work. func TestConcurrentReapersAreBenign(t *testing.T) { diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index ba7c01559..ddc4dc18f 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -1287,9 +1287,14 @@ func (r *issueSQLRepositoryImpl) UnclaimIssueIfAssignee(ctx context.Context, id, // (time.Now().UTC()), same TTL resolution (issueops.LeaseTTL), and the same // only-current-owner classification (storage.ErrAlreadyClaimed / // ErrNotClaimable) — so classic `bd reclaim` staleness semantics see proxied -// heartbeats identically. Deliberately NO Dolt commit: the leases table is -// dolt_ignored (bd-lrgn1), and the cmd layer commits this transaction with -// uow.RunTxEphemeral (plain SQL COMMIT, nothing in dolt_log). +// heartbeats identically. Deliberately NO Dolt commit here: a refresh writes +// only the dolt_ignored leases table (bd-lrgn1). A re-arm is recorded as a +// mutation (versioned rows), so a caller must commit this unit of work with +// a message, as the Heartbeater role's uow leg does (uow.RunTxScoped). A +// refusal is returned as an error, so the caller's unit of work rolls back +// and the orphaned-lease delete the refusal made is not kept on this door +// (no production caller reaches it; the role's leg commits refusals, and the +// live-claim predicate keeps an orphan from ever answering a refresh). func (r *issueSQLRepositoryImpl) HeartbeatIssue(ctx context.Context, id, actor string) error { if issueops.IsActiveWispInTx(ctx, r.runner, id) { return fmt.Errorf("db: IssueSQLRepository.HeartbeatIssue: %w: %s is ephemeral", storage.ErrNotClaimable, id) @@ -1324,6 +1329,13 @@ func (r *issueSQLRepositoryImpl) ReclaimExpiredLeases(ctx context.Context, older return out, nil } +// RenewLease runs the shared heartbeat body directly on r.runner, Reclaim's +// bare-passthrough shape: validation, the wisp refusal and the typed +// refusals all happen inside issueops.ExecuteHeartbeatInTx. +func (r *issueSQLRepositoryImpl) RenewLease(ctx context.Context, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) { + return issueops.ExecuteHeartbeatInTx(ctx, r.runner, request) +} + // Reclaim runs the shared sweep body directly on r.runner, matching GetMany's // bare-passthrough shape: validation and error-wrapping both happen inside // issueops.ExecuteReclaimInTx. diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index 716810eca..0969c8e42 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -122,6 +122,12 @@ type IssueSQLRepository interface { // the same function the two store backends wrap. It is the publicops-typed // counterpart to ReclaimExpiredLeases above, the way GetMany is to GetByIDs. Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) + // RenewLease runs the SHARED heartbeat body (issueops.ExecuteHeartbeatInTx) + // on this repository's transaction, which is how the unit-of-work provider + // reaches the same function the two store backends wrap. It is the + // publicops-typed counterpart to HeartbeatIssue above, the way Reclaim is + // to ReclaimExpiredLeases. + RenewLease(ctx context.Context, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) WakeExpiredDefers(ctx context.Context) (issues, wisps int, err error) } @@ -353,6 +359,14 @@ type IssueUseCase interface { // IssueRepository.Reclaim for why it is a separate method from // ReclaimExpiredLeases rather than a replacement for it. Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) + // RenewLease is the shape issueops.Heartbeater publishes; see + // IssueSQLRepository.RenewLease for why it is a separate method from + // Heartbeat rather than a replacement for it. A refresh writes only the + // ephemeral leases table, but a re-arm (HeartbeatResult.Rearmed) writes + // versioned rows, so the caller commits with uow.RunTxScoped and the + // tables issueops.LeaseRearmVersionCommit names, as the uow Heartbeater + // does — never with uow.RunTxEphemeral. + RenewLease(ctx context.Context, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) WakeExpiredDefers(ctx context.Context) (issues, wisps int, err error) CreateIssue(ctx context.Context, params CreateIssueParams, actor string) (CreateIssueResult, error) @@ -1954,10 +1968,13 @@ func (u *issueUseCaseImpl) UnclaimIfAssignee(ctx context.Context, id, actor, exp return nil } -// Heartbeat refreshes the lease on an issue actor holds in_progress. The -// write touches ONLY the ephemeral leases table (bd-lrgn1), so the caller -// must run it under uow.RunTxEphemeral's no-Dolt-commit form — a heartbeat -// mints no Dolt commit and no history in any mode (bd-aq0ql). +// Heartbeat refreshes the lease on an issue actor holds in_progress. A +// refresh touches ONLY the ephemeral leases table (bd-lrgn1) and needs no Dolt +// commit. A re-arm (a live claim with no lease row the caller holds) is +// recorded as a mutation in versioned tables, and this method cannot report +// one, so never commit it with uow.RunTxEphemeral: take RenewLease, which +// reports the re-arm, and commit as the Heartbeater role does +// (uow.RunTxScoped with issueops.LeaseRearmVersionCommit). func (u *issueUseCaseImpl) Heartbeat(ctx context.Context, id, actor string) error { if id == "" { return fmt.Errorf("Heartbeat: id must not be empty") @@ -1997,3 +2014,11 @@ func (u *issueUseCaseImpl) ReclaimExpiredLeases(ctx context.Context, olderThan t func (u *issueUseCaseImpl) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { return u.issueRepo.Reclaim(ctx, request) } + +// RenewLease passes straight through to the repository with no pre-check and +// no error-wrapping, Reclaim's reasoning: the request is fully validated inside +// the shared body (issueops.ExecuteHeartbeatInTx), and the role's typed +// refusals must reach the caller as the body built them. +func (u *issueUseCaseImpl) RenewLease(ctx context.Context, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) { + return u.issueRepo.RenewLease(ctx, request) +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index ef6e5f32d..4661e34c5 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "federation.go", "get_issue.go", "graph_counter.go", + "heartbeater.go", "init_verifier.go", "issue_claimer.go", "issue_operations.go", @@ -153,6 +154,7 @@ go_test( "git_ref_pin_test.go", "graph_counter_contract_test.go", "graph_walk_plan_test.go", + "heartbeater_contract_test.go", "history_presence_contract_test.go", "history_test.go", "is_blocked_batch_test.go", diff --git a/internal/storage/embeddeddolt/heartbeater.go b/internal/storage/embeddeddolt/heartbeater.go new file mode 100644 index 000000000..d800d2900 --- /dev/null +++ b/internal/storage/embeddeddolt/heartbeater.go @@ -0,0 +1,60 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// Heartbeater returns the lease-refresh surface for this store. +func (s *EmbeddedDoltStore) Heartbeater() (issueops.Heartbeater, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "Heartbeater", Backend: "nil"} + } + return &heartbeater{store: s}, nil +} + +// heartbeater answers a Heartbeat from one connection's transaction around the +// shared body, storeops.ExecuteHeartbeatInTx, the embedded twin of the +// server-backed store's heartbeater. A REFRESH commits as SQL only: the +// leases table is dolt_ignored, so there is nothing versioned to record and +// no version-control entry follows it. A RE-ARM is recorded as a mutation and +// gets the version-control entry LeaseRearmVersionCommit names. +type heartbeater struct{ store *EmbeddedDoltStore } + +var _ issueops.Heartbeater = (*heartbeater)(nil) + +func (h *heartbeater) Heartbeat(ctx context.Context, request issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + if err := storeops.ValidateHeartbeatRequest(request); err != nil { + return issueops.HeartbeatResult{}, err + } + var result issueops.HeartbeatResult + var refusal error + if err := h.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + result, err = storeops.ExecuteHeartbeatInTx(ctx, tx, request) + // A lease refusal commits (it may have deleted an orphaned lease + // row) and is reported after the commit; see HeartbeatRefusalCommits. + refusal = nil + if storeops.HeartbeatRefusalCommits(err) { + refusal = err + return nil, "", nil + } + if err != nil { + return nil, "", err + } + tables, msg := storeops.LeaseRearmVersionCommit(request.IssueID, result.Rearmed) + return tables, msg, nil + }); err != nil { + return issueops.HeartbeatResult{}, err + } + if refusal != nil { + return issueops.HeartbeatResult{}, refusal + } + return result, nil +} diff --git a/internal/storage/embeddeddolt/heartbeater_contract_test.go b/internal/storage/embeddeddolt/heartbeater_contract_test.go new file mode 100644 index 000000000..fd96ffcbe --- /dev/null +++ b/internal/storage/embeddeddolt/heartbeater_contract_test.go @@ -0,0 +1,139 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" +) + +// TestHeartbeaterContract runs the Heartbeater contract against the embedded +// store, which wraps internal/storage/issueops.ExecuteHeartbeatInTx in one +// connection's transaction and records no version entry after a refresh, one +// after a re-arm. +func TestHeartbeaterContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "hbc") + ctx := t.Context() + fixture := newEmbeddedHeartbeaterFixture(t, te, "hbc") + t.Run("RefreshesTheHoldersLease", func(t *testing.T) { + conformance.RunHeartbeaterRefreshesTheHoldersLease(t, ctx, fixture) + }) + t.Run("ChangesNothingOnTheRow", func(t *testing.T) { + conformance.RunHeartbeaterChangesNothingOnTheRow(t, ctx, fixture) + }) + t.Run("AGuardedWriteWithThePreHeartbeatRevisionStillWins", func(t *testing.T) { + conformance.RunHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins(t, ctx, fixture) + }) + t.Run("RefusesAForeignHolder", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAForeignHolder(t, ctx, fixture) + }) + t.Run("RefusesAfterClose", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterClose(t, ctx, fixture) + }) + t.Run("RefusesAfterRelease", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterRelease(t, ctx, fixture) + }) + t.Run("RefusesAfterReclaim", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterReclaim(t, ctx, fixture) + }) + t.Run("RefusesAfterStatusMovedOffInProgress", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterStatusMovedOffInProgress(t, ctx, fixture) + }) + t.Run("RefusesAWisp", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAWisp(t, ctx, fixture) + }) + t.Run("RearmsAHandDoledClaim", func(t *testing.T) { + conformance.RunHeartbeaterRearmsAHandDoledClaim(t, ctx, fixture) + }) + t.Run("AnswersNotFoundForAnAbsentID", func(t *testing.T) { + conformance.RunHeartbeaterAnswersNotFoundForAnAbsentID(t, ctx, fixture) + }) + t.Run("RefusesABlankOrOverLongActor", func(t *testing.T) { + conformance.RunHeartbeaterRefusesABlankOrOverLongActor(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheRequest", func(t *testing.T) { + conformance.RunHeartbeaterDoesNotMutateTheRequest(t, ctx, fixture) + }) + t.Run("CommitsOnlyItsOwnTables", func(t *testing.T) { + conformance.RunHeartbeaterCommitsOnlyItsOwnTables(t, ctx, fixture) + }) + t.Run("ASameSecondHeartbeatIsNotARearm", func(t *testing.T) { + conformance.RunHeartbeaterASameSecondHeartbeatIsNotARearm(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAReleasedIssue", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAClosedIssue", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAClosedIssue(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds(t, ctx, fixture) + }) +} + +func newEmbeddedHeartbeaterFixture(t *testing.T, te *testEnv, prefix string) conformance.HeartbeaterFixture { + t.Helper() + heartbeater, err := te.store.Heartbeater() + if err != nil { + t.Fatalf("Heartbeater(): %v", err) + } + claimer, err := te.store.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := te.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + releaser, err := te.store.Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + reclaimer, err := te.store.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := heartbeater + if runner != nil { + if hooked, err = storage.NewHookFiringStore(te.store, runner).Heartbeater(); err != nil { + t.Fatalf("hooked Heartbeater(): %v", err) + } + } + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.HeartbeaterFixture{ + IssuePrefix: kit.IssuePrefix, + Heartbeater: heartbeater, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + Claimer: claimer, + Lifecycle: lifecycle, + Releaser: releaser, + LeaseReclaimer: reclaimer, + QueryScalar: kit.QueryScalar, + CountHistory: kit.CountHistory, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + for _, stmt := range statements { + if _, err := db.ExecContext(ctx, stmt.Query, stmt.Args...); err != nil { + return fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return nil + }, + SetJournalEnabled: te.store.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: te.store.SetVersionedHistoryEnabled, + HookedHeartbeater: hooked, + FiredUpdates: fired, + } +} diff --git a/internal/storage/embeddeddolt/issues.go b/internal/storage/embeddeddolt/issues.go index 2f52b99b7..4ef78aff1 100644 --- a/internal/storage/embeddeddolt/issues.go +++ b/internal/storage/embeddeddolt/issues.go @@ -113,15 +113,36 @@ func (s *EmbeddedDoltStore) UpdateIssueChecked(ctx context.Context, id string, u } // HeartbeatIssue refreshes the lease on an issue actor holds in_progress. -// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +// Delegates SQL work to issueops. A REFRESH writes only the dolt_ignored +// leases row, so it stages nothing and mints no Dolt commit; a RE-ARM is +// recorded as a mutation and gets the one version-control commit +// issueops.LeaseRearmVersionCommit names. func (s *EmbeddedDoltStore) HeartbeatIssue(ctx context.Context, id, actor string) error { - return s.withConn(ctx, true, func(tx *sql.Tx) error { + // A refusal that read the issue row COMMITS (it may have deleted the + // caller's orphaned lease row) and is returned after; one whose issue read + // failed is returned from the callback, so its transaction rolls back + // (issueops.LeaseRenewalRefusalCommits). + var refusal error + if err := s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (issueops.ChangedTables, string, error) { + refusal = nil if issueops.IsActiveWispInTx(ctx, tx, id) { // Wisps are ephemeral and never leased; nothing to heartbeat. - return fmt.Errorf("%w: %s is ephemeral", storage.ErrNotClaimable, id) + return nil, "", fmt.Errorf("%w: %s is ephemeral", storage.ErrNotClaimable, id) } - return issueops.HeartbeatIssueInTx(ctx, tx, id, actor) - }) + renewal, err := issueops.RenewLeaseInTx(ctx, tx, id, actor) + if issueops.LeaseRenewalRefusalCommits(err) { + refusal = err + return nil, "", nil + } + if err != nil { + return nil, "", err + } + tables, msg := issueops.LeaseRearmVersionCommit(id, renewal.Rearmed) + return tables, msg, nil + }); err != nil { + return err + } + return refusal } // ReclaimExpiredLeases reverts in_progress issues whose lease expired more than diff --git a/internal/storage/embeddeddolt/lease_test.go b/internal/storage/embeddeddolt/lease_test.go index cb712b1a8..35ff468d4 100644 --- a/internal/storage/embeddeddolt/lease_test.go +++ b/internal/storage/embeddeddolt/lease_test.go @@ -190,8 +190,9 @@ func TestReclaimExpiredLeaseSurvivesRestartEmbedded(t *testing.T) { // claimed under, and still refuses a genuinely different identity. The // primary UPDATE's own holder=? predicate stays verbatim SQL by design (see // lease.go); this test exercises the fallback path that makes the caller's -// experience correct anyway, including that it re-arms the lease row under -// actor's current spelling so the NEXT heartbeat hits the fast path. +// experience correct anyway, including that it refreshes the lease row in +// place under actor's current spelling, keeping its grant (granted_at and +// granted_node), rather than re-granting it. func TestHeartbeatMatchesActorAcrossSpellingEmbedded(t *testing.T) { skipUnlessEmbeddedDolt(t) @@ -211,6 +212,12 @@ func TestHeartbeatMatchesActorAcrossSpellingEmbedded(t *testing.T) { if err := te.store.ClaimIssue(ctx, "lease-spelling-1", "gastown.mayor"); err != nil { t.Fatalf("ClaimIssue: %v", err) } + // Give the grant a provenance a re-grant would visibly overwrite (it + // stamps granted_at = now and this node, which is unnamed here). + te.exec(t, ctx, "UPDATE leases SET granted_at = DATE_SUB(granted_at, INTERVAL 1 HOUR), granted_node = 'replica-b' WHERE issue_id = ?", "lease-spelling-1") + const grantQuery = "SELECT CONCAT(CAST(granted_at AS CHAR), '|', COALESCE(granted_node, '')) FROM leases WHERE issue_id = ?" + var grantBefore string + te.queryScalar(t, ctx, grantQuery, []any{"lease-spelling-1"}, &grantBefore) // Same identity, dot-separator form sanitized to double-underscore (the // exact ga-wzl83 repro shape) — must succeed, not ErrAlreadyClaimed. @@ -218,13 +225,18 @@ func TestHeartbeatMatchesActorAcrossSpellingEmbedded(t *testing.T) { t.Fatalf("cross-spelling HeartbeatIssue: %v, want success (same identity, different separator)", err) } - // The re-arm re-wrote holder under the caller's current spelling, so the - // NEXT heartbeat under that same spelling hits the primary UPDATE's fast - // path directly (0 rows would mean it fell to the fallback again). - var holder string + // The in-place refresh re-wrote holder under the caller's current + // spelling and kept the grant. The assignee keeps the claim's spelling, so + // every later heartbeat under this one still fails the fast path's + // verbatim live-claim check and is refreshed in place the same way. + var holder, grantAfter string te.queryScalar(t, ctx, "SELECT holder FROM leases WHERE issue_id = ?", []any{"lease-spelling-1"}, &holder) if holder != "gastown__mayor" { - t.Errorf("lease holder after cross-spelling heartbeat = %q, want re-armed to %q", holder, "gastown__mayor") + t.Errorf("lease holder after cross-spelling heartbeat = %q, want refreshed in place under %q", holder, "gastown__mayor") + } + te.queryScalar(t, ctx, grantQuery, []any{"lease-spelling-1"}, &grantAfter) + if grantAfter != grantBefore { + t.Errorf("granted_at|granted_node moved from %s to %s across a cross-spelling heartbeat; a refresh never re-grants", grantBefore, grantAfter) } // Control: a genuinely different identity — canonicalActor's own doc @@ -257,3 +269,87 @@ func TestHeartbeatRejectsWispEmbedded(t *testing.T) { t.Fatalf("wisp heartbeat err = %v, want ErrNotClaimable", err) } } + +// TestHeartbeatIssueCommitsARefusalOnlyWhenItReadTheIssueEmbedded is the +// embedded twin of the server-backed store's test: a refusal whose issue read +// failed rolls back the fast path's extension of a lease row with no issue +// row, and a refusal that read the issue row commits its orphan delete +// (issueops.LeaseRenewalRefusalCommits), which takes the caller's orphan under +// any spelling of its identity and never another identity's. +func TestHeartbeatIssueCommitsARefusalOnlyWhenItReadTheIssueEmbedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + te := newTestEnv(t, "lease_refusal") + ctx := t.Context() + + plantLease := func(id, holder string) { + t.Helper() + granted := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + te.exec(t, ctx, ` + INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at, granted_node) + VALUES (?, ?, ?, ?, ?, '') + `, id, holder, granted, granted.Add(time.Minute), granted) + } + seedOpenIssue := func(id string) { + t.Helper() + if err := te.store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "orphaned lease", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seeder"); err != nil { + t.Fatalf("CreateIssue %s: %v", id, err) + } + } + leaseRows := func(id string) int { + t.Helper() + var rows int + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", []any{id}, &rows) + return rows + } + hbCtx := issueops.WithLeaseTTL(ctx, time.Hour) + + // No issue row: the refusal read nothing, so the fast path's extension + // must roll back. + plantLease("lease-noissue-1", "alice") + var before, after string + te.queryScalar(t, ctx, "SELECT CAST(lease_expires_at AS CHAR) FROM leases WHERE issue_id = ?", []any{"lease-noissue-1"}, &before) + if err := te.store.HeartbeatIssue(hbCtx, "lease-noissue-1", "alice"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("heartbeat on a lease row with no issue row err = %v, want ErrNotClaimable", err) + } + te.queryScalar(t, ctx, "SELECT CAST(lease_expires_at AS CHAR) FROM leases WHERE issue_id = ?", []any{"lease-noissue-1"}, &after) + if after != before { + t.Errorf("refused heartbeat committed the fast path's extension: lease_expires_at %s -> %s, want unchanged", before, after) + } + + // An open issue: the refusal read it and deleted the caller's orphaned + // row, and that delete must commit. + seedOpenIssue("lease-orphan-1") + plantLease("lease-orphan-1", "alice") + if err := te.store.HeartbeatIssue(hbCtx, "lease-orphan-1", "alice"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("heartbeat over an orphaned lease on an open issue err = %v, want ErrNotClaimable", err) + } + if rows := leaseRows("lease-orphan-1"); rows != 0 { + t.Errorf("lease rows on lease-orphan-1 after the refusal = %d, want 0: the refusal's orphan delete must commit", rows) + } + + // The caller's orphan held under another spelling of its identity + // (ga-v2k49): the delete's holder fence is judged under actorMatches, as + // the in-place refresh's is, so the row is the caller's and is deleted. + seedOpenIssue("lease-orphan-spelling-1") + plantLease("lease-orphan-spelling-1", "gastown.mayor") + if err := te.store.HeartbeatIssue(hbCtx, "lease-orphan-spelling-1", "gastown__mayor"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("cross-spelling heartbeat over an orphaned lease err = %v, want ErrNotClaimable", err) + } + if rows := leaseRows("lease-orphan-spelling-1"); rows != 0 { + t.Errorf("lease rows on lease-orphan-spelling-1 after a cross-spelling refusal = %d, want 0: the orphan is the caller's under another spelling", rows) + } + + // Control: a genuinely different identity's orphan is not the caller's, + // so the refusal leaves it alone. + seedOpenIssue("lease-orphan-other-1") + plantLease("lease-orphan-other-1", "gastown.dog-3") + if err := te.store.HeartbeatIssue(hbCtx, "lease-orphan-other-1", "gastown__mayor"); !errors.Is(err, storage.ErrNotClaimable) { + t.Fatalf("heartbeat over another identity's orphaned lease err = %v, want ErrNotClaimable", err) + } + if rows := leaseRows("lease-orphan-other-1"); rows != 1 { + t.Errorf("lease rows on lease-orphan-other-1 after the refusal = %d, want 1: a refusal deletes only the caller's own row", rows) + } +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 4467e13d8..3089877d1 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -144,7 +144,11 @@ func (p *uowProvider) Releaser() (publicops.Releaser, error) { return uow. func (p *uowProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { return uow.NewLeaseReclaimer(p) } -func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } + +// Heartbeater passes through: the external-dependency policy governs what a +// write may point at, and a heartbeat on an existing claim points at nothing. +func (p *uowProvider) Heartbeater() (publicops.Heartbeater, error) { return uow.NewHeartbeater(p) } +func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } func (p *uowProvider) EventsJournalCursor() (storage.EventsJournalCursor, error) { return uow.NewEventsJournalCursor(p) } @@ -222,6 +226,7 @@ var ( _ uow.MetadataCASSource = (*uowProvider)(nil) _ uow.ReleaserSource = (*uowProvider)(nil) _ uow.LeaseReclaimerSource = (*uowProvider)(nil) + _ uow.HeartbeaterSource = (*uowProvider)(nil) _ uow.MemoriesSource = (*uowProvider)(nil) _ uow.EventsJournalCursorSource = (*uowProvider)(nil) ) diff --git a/internal/storage/hook_batch_applier_test.go b/internal/storage/hook_batch_applier_test.go index a211e0908..84bbbff72 100644 --- a/internal/storage/hook_batch_applier_test.go +++ b/internal/storage/hook_batch_applier_test.go @@ -42,6 +42,9 @@ func (r *recordingBatchApplyHooks) CompleteIssueOperationRelease(_ context.Conte func (r *recordingBatchApplyHooks) CompleteIssueOperationReclaim(_ context.Context, issueID string) { r.completions = append(r.completions, "reclaim:"+issueID) } +func (r *recordingBatchApplyHooks) CompleteIssueOperationUpdateByID(_ context.Context, issueID string) { + r.completions = append(r.completions, "update-by-id:"+issueID) +} func issueIDOrNil(issue *types.Issue) string { if issue == nil { @@ -234,3 +237,4 @@ func (r *recordingBatchApplyIssues) CompleteIssueOperationComment(context.Contex func (r *recordingBatchApplyIssues) CompleteIssueOperationMetadata(context.Context, string) {} func (r *recordingBatchApplyIssues) CompleteIssueOperationRelease(context.Context, string) {} func (r *recordingBatchApplyIssues) CompleteIssueOperationReclaim(context.Context, string) {} +func (r *recordingBatchApplyIssues) CompleteIssueOperationUpdateByID(context.Context, string) {} diff --git a/internal/storage/hook_decorator.go b/internal/storage/hook_decorator.go index 733038d33..4a352736a 100644 --- a/internal/storage/hook_decorator.go +++ b/internal/storage/hook_decorator.go @@ -160,6 +160,11 @@ func RoleFiresHooks(role any) bool { // or a request. case *hookLeaseReclaimer: return true + // The heartbeater fires the update hook for a RE-ARM only + // (hook_heartbeater.go): rare, but a write the workspace's script runs + // for, so a server handed an unpeeled one is refused like the others. + case *hookHeartbeater: + return true } return false } @@ -482,6 +487,13 @@ func (h *HookFiringStore) CompleteIssueOperationReclaim(ctx context.Context, iss h.fireHookByID(ctx, hooks.EventUpdate, issueID) } +// CompleteIssueOperationUpdateByID fires on_update for a write that names +// its row by id (a heartbeat's re-arm), reading the row back to hand it to +// the script. +func (h *HookFiringStore) CompleteIssueOperationUpdateByID(ctx context.Context, issueID string) { + h.fireHookByID(ctx, hooks.EventUpdate, issueID) +} + func (h *HookFiringStore) fireHookByID(ctx context.Context, event, id string) { if !h.hookConfigured(event) { return diff --git a/internal/storage/hook_heartbeater.go b/internal/storage/hook_heartbeater.go new file mode 100644 index 000000000..84edb18aa --- /dev/null +++ b/internal/storage/hook_heartbeater.go @@ -0,0 +1,47 @@ +package storage + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// Heartbeater returns the inner store's lease-refresh surface with this +// decorator's completion hook layered over it. +// +// A REFRESH FIRES NOTHING. It writes only the ephemeral lease row: it changes +// no field a hook script could read back, it is the hottest write in a fleet, +// and the raw HeartbeatIssue has never fired a hook. Firing on_update for it +// would run a user's subprocess every ninety seconds per worker for a row +// that did not change. +// +// A RE-ARM FIRES on_update, once. It is recorded as a mutation (row_lock, +// updated_at, an event, a journal entry, a version), the row it names is +// still there to hand a script, and the result's Rearmed is the fact that the +// row was written — the Releaser's Changed rule. +// +// It recurses instead of delegating, so a blind delegation would hand back +// the inner store's surface and silently stop firing it. +func (h *HookFiringStore) Heartbeater() (issueops.Heartbeater, error) { + inner, err := h.inner.Heartbeater() + if err != nil { + return nil, err + } + return &hookHeartbeater{inner: inner, hooks: h}, nil +} + +type hookHeartbeater struct { + inner issueops.Heartbeater + hooks issueOperationHooks +} + +// Heartbeat fires the update hook for a re-arm and nothing for a refresh or +// a refusal. +func (b *hookHeartbeater) Heartbeat(ctx context.Context, request issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + result, err := b.inner.Heartbeat(ctx, request) + if err != nil || !result.Rearmed { + return result, err + } + b.hooks.CompleteIssueOperationUpdateByID(ctx, result.IssueID) + return result, nil +} diff --git a/internal/storage/hook_heartbeater_test.go b/internal/storage/hook_heartbeater_test.go new file mode 100644 index 000000000..bed58c5f1 --- /dev/null +++ b/internal/storage/hook_heartbeater_test.go @@ -0,0 +1,42 @@ +package storage + +import ( + "context" + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +type stubHeartbeater struct{ result issueops.HeartbeatResult } + +func (s stubHeartbeater) Heartbeat(context.Context, issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + return s.result, nil +} + +// TestHookHeartbeaterFiresOnlyForARearm pins hook_heartbeater.go: a refresh +// fires nothing, a re-arm (a mutation) fires on_update once. +func TestHookHeartbeaterFiresOnlyForARearm(t *testing.T) { + for _, test := range []struct { + name string + rearmed bool + want []string + }{ + {"a refresh fires nothing", false, nil}, + {"a re-arm fires on_update", true, []string{"update-by-id:bd-1"}}, + } { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingIssueOperationHooks{} + hooked := &hookHeartbeater{ + inner: stubHeartbeater{result: issueops.HeartbeatResult{IssueID: "bd-1", Rearmed: test.rearmed}}, + hooks: recorder, + } + if _, err := hooked.Heartbeat(t.Context(), issueops.HeartbeatRequest{Actor: "w", IssueID: "bd-1"}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(recorder.completions, test.want) { + t.Fatalf("completions = %v, want %v", recorder.completions, test.want) + } + }) + } +} diff --git a/internal/storage/hook_issue_operations.go b/internal/storage/hook_issue_operations.go index 2b963ebbe..9dbcf7278 100644 --- a/internal/storage/hook_issue_operations.go +++ b/internal/storage/hook_issue_operations.go @@ -36,6 +36,7 @@ type issueOperationHooks interface { CompleteIssueOperationMetadata(ctx context.Context, issueID string) CompleteIssueOperationRelease(ctx context.Context, issueID string) CompleteIssueOperationReclaim(ctx context.Context, issueID string) + CompleteIssueOperationUpdateByID(ctx context.Context, issueID string) } var _ issueOperationHooks = (*HookFiringStore)(nil) diff --git a/internal/storage/hook_issue_operations_test.go b/internal/storage/hook_issue_operations_test.go index 13a788060..77eb0b359 100644 --- a/internal/storage/hook_issue_operations_test.go +++ b/internal/storage/hook_issue_operations_test.go @@ -66,6 +66,9 @@ func (r *recordingIssueOperationHooks) CompleteIssueOperationRelease(_ context.C func (r *recordingIssueOperationHooks) CompleteIssueOperationReclaim(_ context.Context, issueID string) { r.completions = append(r.completions, "reclaim:"+issueID) } +func (r *recordingIssueOperationHooks) CompleteIssueOperationUpdateByID(_ context.Context, issueID string) { + r.completions = append(r.completions, "update-by-id:"+issueID) +} // lifecycleStore is a DoltStorage whose only real method is IssueLifecycle. type lifecycleStore struct { diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 90d91f02b..3768c802c 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -58,6 +58,7 @@ go_library( "filters.go", "gate_fields.go", "get_issue.go", + "heartbeat_role.go", "helpers.go", "history.go", "history_presence.go", diff --git a/internal/storage/issueops/aggregate.go b/internal/storage/issueops/aggregate.go index 36ec44b57..047c4e763 100644 --- a/internal/storage/issueops/aggregate.go +++ b/internal/storage/issueops/aggregate.go @@ -25,6 +25,19 @@ func (tables ChangedTables) Add(names ...string) { } } +// Sorted returns the recorded tables in name order (nil when empty). +func (tables ChangedTables) Sorted() []string { + if len(tables) == 0 { + return nil + } + out := make([]string, 0, len(tables)) + for table := range tables { + out = append(out, table) + } + sort.Strings(out) + return out +} + // Merge records every table in other. func (tables ChangedTables) Merge(other map[string]bool) { for table := range other { diff --git a/internal/storage/issueops/heartbeat_role.go b/internal/storage/issueops/heartbeat_role.go new file mode 100644 index 000000000..646d652ff --- /dev/null +++ b/internal/storage/issueops/heartbeat_role.go @@ -0,0 +1,135 @@ +package issueops + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// ValidateHeartbeatRequest applies the request rules every Heartbeater +// implementation shares. It is the ONE definition of a valid heartbeat: the +// library legs run it before they open a transaction, and the shared body runs +// it again, so a leg that skipped it still answers the same contract. +// +// THE ACTOR IS NOT TRIMMED, Releaser.Actor's rule: it is the fence's subject +// and is compared with the lease row's holder as sent. Only a blank one is +// refused, and one longer than the lease row's holder column can hold. +func ValidateHeartbeatRequest(request publicops.HeartbeatRequest) error { + if strings.TrimSpace(request.Actor) == "" { + return heartbeatFieldError(publicops.HeartbeatFieldActor, "heartbeat actor is required") + } + if err := types.CheckFieldLen("actor", request.Actor); err != nil { + return heartbeatFieldError(publicops.HeartbeatFieldActor, err.Error()) + } + if request.IssueID == "" { + return heartbeatFieldError(publicops.HeartbeatFieldIssueID, "heartbeat issue id is required") + } + return nil +} + +func heartbeatFieldError(field, detail string) error { + return &publicops.HeartbeatFieldError{Field: field, Detail: storage.ErrValidation.Error() + ": " + detail} +} + +// ExecuteHeartbeatInTx refreshes the lease request.Actor holds on +// request.IssueID, in tx. It is the body behind the Heartbeater accessor on +// ALL THREE library legs: dolt, embedded dolt and the unit-of-work provider. +// +// It is RenewLeaseInTx — the native heartbeat, SQL text unchanged — with the +// role's request rules in front and its typed refusals behind: +// +// - a wisp is refused before anything is written, as every native leg +// refuses it ("is ephemeral"): wisps are never leased; +// - an id that names no row on either plane is storage.ErrNotFound, where +// the native slow path answered ErrNotClaimable; +// - a foreign holder and a missing live claim are *LeaseConflictError, +// classified from the issue row the slow path already read. +// +// A REFUSAL MAY WRITE ONE THING: it deletes the caller's own lease row when +// that row survives on an issue the caller no longer holds a live claim on (an +// orphan; see RenewLeaseInTx). The legs therefore COMMIT the transaction of a +// refusal HeartbeatRefusalCommits recognizes and report the refusal after the +// commit, so the cleanup is not rolled back with it. +// +// WHAT A REFRESH MUST NEVER DO is everything RenewLeaseInTx's own doc rules +// out, and the conformance contract asserts each one: remint row_lock, record +// a version (current_revision, issue_versions), stamp updated_at, write an +// event or a journal entry. The legs add the other two: no version-control +// commit and no hook. A RE-ARM (HeartbeatResult.Rearmed) does every one of +// them, because it is recorded as a mutation: the legs commit it once, as +// LeaseRearmVersionCommit names, and the hook decorator fires on_update. +func ExecuteHeartbeatInTx(ctx context.Context, tx DBTX, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) { + if err := ValidateHeartbeatRequest(request); err != nil { + return publicops.HeartbeatResult{}, err + } + id := request.IssueID + + var wispStatus string + if err := tx.QueryRowContext(ctx, "SELECT status FROM wisps WHERE id = ? LIMIT 1", id).Scan(&wispStatus); err == nil { + return publicops.HeartbeatResult{}, &publicops.LeaseConflictError{ + IssueID: id, + Status: types.Status(wispStatus), + Ephemeral: true, + Err: fmt.Errorf("%w: %s is ephemeral", publicops.ErrLeaseNotHeld, id), + } + } + + renewal, err := RenewLeaseInTx(ctx, tx, id, request.Actor) + if err != nil { + var refusal *LeaseRenewalRefusal + if !errors.As(err, &refusal) { + return publicops.HeartbeatResult{}, err + } + return publicops.HeartbeatResult{}, classifyLeaseRenewalRefusal(refusal, request.Actor) + } + return publicops.HeartbeatResult{ + IssueID: id, + Holder: request.Actor, + HeartbeatAt: renewal.HeartbeatAt, + LeaseExpiresAt: renewal.ExpiresAt, + Rearmed: renewal.Rearmed, + }, nil +} + +// classifyLeaseRenewalRefusal types a native heartbeat refusal for the role. +func classifyLeaseRenewalRefusal(refusal *LeaseRenewalRefusal, actor string) error { + id := refusal.IssueID + if !refusal.Found { + if errors.Is(refusal.ReadErr, sql.ErrNoRows) { + return fmt.Errorf("%w: issue %s", storage.ErrNotFound, id) + } + return fmt.Errorf("heartbeat %s: read issue: %w", id, refusal.ReadErr) + } + status := types.Status(refusal.Status) + if refusal.Wisp { + return &publicops.LeaseConflictError{ + IssueID: id, Status: status, Ephemeral: true, + Err: fmt.Errorf("%w: %s is ephemeral", publicops.ErrLeaseNotHeld, id), + } + } + if refusal.Assignee != "" && !actorMatches(refusal.Assignee, actor) { + return &publicops.LeaseConflictError{ + IssueID: id, Holder: refusal.Assignee, Status: status, Err: refusal.Err, + } + } + return &publicops.LeaseConflictError{ + IssueID: id, Status: status, + Err: fmt.Errorf("%w: %s status %s", publicops.ErrLeaseNotHeld, id, refusal.Status), + } +} + +// HeartbeatRefusalCommits reports whether err is a heartbeat refusal whose +// transaction a leg must COMMIT rather than roll back: a typed +// *LeaseConflictError, the one refusal that can carry a write (the orphaned +// lease row ExecuteHeartbeatInTx deletes). Committing a refusal that wrote +// nothing is a no-op. Every other error rolls back as usual. +func HeartbeatRefusalCommits(err error) bool { + var conflict *publicops.LeaseConflictError + return errors.As(err, &conflict) +} diff --git a/internal/storage/issueops/journal_completeness_test.go b/internal/storage/issueops/journal_completeness_test.go index c33ff8ef4..5fa0c9451 100644 --- a/internal/storage/issueops/journal_completeness_test.go +++ b/internal/storage/issueops/journal_completeness_test.go @@ -98,7 +98,16 @@ var mutationEntryPoints = []string{ "ClaimReadyIssueInTx", "UnclaimIssueInTx", "UnclaimIssueIfAssigneeInTx", - // HeartbeatIssueInTx is deliberately absent — see journalExemptMutations. + // The heartbeat family journals on its RE-ARM path only (a live claim + // with no lease row the caller holds: row_lock reminted, a lease granted, + // recorded as a mutation). Its REFRESH path writes only the clone-local + // leases table and still records nothing; the fast-path return in + // RenewLeaseInTx says why. The ruling that exempted the whole family + // changed when the re-arm became a mutation (it must collide with a racing + // release or close on row_lock), so the family is listed here. + "RenewLeaseInTx", + "HeartbeatIssueInTx", + "ExecuteHeartbeatInTx", "ReclaimExpiredLeasesInTx", // plane moves, graph edges, labels, comments, renames "PromoteFromEphemeralInTx", @@ -209,13 +218,7 @@ var beadDMLExemptions = map[string]string{ // never are. TestExemptMutationsDoNotJournal pins the decision from the other // side — each must exist and must NOT emit — so re-adding an emit fails loudly // instead of silently reversing the ruling. -var journalExemptMutations = map[string]string{ - "HeartbeatIssueInTx": "high-frequency lease keepalive: writes only the clone-local `leases` table " + - "(lease-liveness state), never a durable bead field. Journaling it would put a full-snapshot " + - "write plus the shared seq-counter serialization on the hottest write in the fleet, and a replay " + - "consumer gains nothing — lease state is working-set-plane and expires on its own. Lease RECLAIM " + - "is the opposite case and does journal: it clears assignee and reverts status, which is durable.", -} +var journalExemptMutations = map[string]string{} // TestEveryMutationFunctionJournals parses this package's source, builds the // intra-package call graph, and asserts every mutation entry point either diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go index 2ec940280..d45582219 100644 --- a/internal/storage/issueops/lease.go +++ b/internal/storage/issueops/lease.go @@ -3,7 +3,9 @@ package issueops import ( "context" "crypto/rand" + "database/sql" "encoding/binary" + "errors" "fmt" "os" "strings" @@ -62,10 +64,14 @@ func leaseTTL(ctx context.Context) time.Duration { // races care about (claim, close, updateIssueInTx, reclaim, unclaim all do). // Paths that touch only orthogonal cells (is_blocked, compaction_level, // dependency metadata, rename, or reopen — which acts on closed rows) are safe -// to merge with a reclaim and intentionally do NOT rewrite it. Heartbeats no -// longer touch the issues row at all (bd-lrgn1): the lease lives in the +// to merge with a reclaim and intentionally do NOT rewrite it. A heartbeat +// REFRESH does not touch the issues row (bd-lrgn1): the lease lives in the // ephemeral leases table, where a racing heartbeat and reclaim contend on the -// SAME lease row and conflict without any help. Adding a new path that sets +// SAME lease row and conflict without any help. A heartbeat RE-ARM (a lease +// granted to a live claim that has none) is the one deliberate remint that +// changes no lifecycle field, and it is load-bearing: its lease INSERT shares +// no row with a concurrent release or close, so the remint is what makes the +// two collide (see RenewLeaseInTx). Adding a new path that sets // status/assignee outside updateIssueInTx without rewriting row_lock would // silently reintroduce the zombie-merge bug. // @@ -78,10 +84,11 @@ func leaseTTL(ctx context.Context) time.Duration { // that already goes out of its way to preserve updated_at // (blocked_state.go), and bumping the token there would clobber a concurrent // whole-row CAS over a cell derived from OTHER rows. (The PR's second -// exemption of this kind, the lease heartbeat, no longer arises here — since -// bd-lrgn1 heartbeats never touch the issues row at all.) So the invariant -// cuts both ways: status/ownership writes MUST remint, aux-marker writes MUST -// NOT. The remint direction is enforced at build time by +// exemption of this kind, the lease heartbeat, no longer arises here: since +// bd-lrgn1 a refresh never touches the issues row, and a re-arm remints on +// purpose, accepting that a guarded write composed before it fails once and +// re-reads.) So the invariant cuts both ways: status/ownership writes MUST +// remint, aux-marker writes MUST NOT. The remint direction is enforced at build time by // TestAllIssueRowWritesStampRowLock (row_lock_guard_test.go), whose exemption // markers are the machine-readable form of this list — widen the stamping // policy there first, not by ad-hoc stamps. @@ -293,14 +300,115 @@ func RestoreLeaseOnImportInTx(ctx context.Context, tx DBTX, issue *types.Issue, // storage.ErrNotClaimable / ErrAlreadyClaimed so the caller learns its lease // is gone. // -// The write touches ONLY the leases table (ephemeral, dolt_ignored): a +// A refresh touches ONLY the leases table (ephemeral, dolt_ignored): a // heartbeat mints no Dolt commit and no history, and deliberately does NOT // stamp issues.updated_at — updated_at keeps its merge/LWW meaning and bd // stale consults leases.heartbeat_at for in_progress rows instead (bd-lrgn1). // -//nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) +// A RE-ARM (a live claim with no lease row the caller holds, which the +// heartbeat grants) is the exception: it is recorded as a mutation (row_lock, +// updated_at, event, journal, version); see RenewLeaseInTx. A caller that runs +// this in a transaction must make the version-control commit +// LeaseRearmVersionCommit names when it re-arms. +// +// It is RenewLeaseInTx with the renewal's report dropped: the legacy +// HeartbeatIssue legs answer only whether the lease was refreshed. func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { + _, err := RenewLeaseInTx(ctx, tx, id, actor) + return err +} + +// LeaseRenewal reports one lease RenewLeaseInTx refreshed or re-armed. Both +// times are computed in Go from the same clock reading and TTL the statement +// binds, rounded to whole seconds as the lease columns round them on store, +// so they are what the lease row now holds. +type LeaseRenewal struct { + HeartbeatAt time.Time + ExpiresAt time.Time + // Rearmed is true only when the slow path CREATED the caller's lease row: + // the caller held a live claim with no lease row for it (a hand-doled + // claim), or the issue's lease row named a holder that is not the caller. + // A refresh of the caller's own row is never a re-arm, even when it + // changed no value (two heartbeats in one second). + Rearmed bool +} + +// LeaseRearmVersionCommit is the version-control commit a leg makes after a +// heartbeat: none for a refresh or a refusal (the leases table is +// dolt_ignored), and the release's staged set, issues and events, for a +// re-arm, which is recorded as a mutation. One helper for every leg so the +// message cannot drift. +func LeaseRearmVersionCommit(id string, rearmed bool) (ChangedTables, string) { + if !rearmed { + return nil, "" + } + tables := ChangedTables{} + tables.Add("issues", "events") + return tables, fmt.Sprintf("bd: re-arm lease %s", id) +} + +// LeaseRenewalRefusal is the refusal RenewLeaseInTx answers when no lease row +// matched the caller and the slow path could not re-arm one. It carries the +// issue row the slow path read so a typed front door (ExecuteHeartbeatInTx) +// can classify the refusal without a second read, and it reads and unwraps as +// Err — the exact error HeartbeatIssueInTx has always returned — so the +// legacy legs' text and errors.Is behavior are unchanged. +type LeaseRenewalRefusal struct { + IssueID string + // Found is false when the issue row could not be read. + Found bool + // ReadErr is the error the issue-row read failed with when Found is false. + ReadErr error + Wisp bool + Assignee string + Status string + // Err is the wrapped refusal: storage.ErrAlreadyClaimed or + // storage.ErrNotClaimable. + Err error + // OrphanDeleted is true when the refusal deleted a lease row the caller + // held on an issue it no longer holds a live claim on (see + // RenewLeaseInTx). The deletion is part of the refusing transaction: a + // caller that wants it kept must COMMIT that transaction and report the + // refusal afterwards (HeartbeatRefusalCommits). + OrphanDeleted bool +} + +func (e *LeaseRenewalRefusal) Error() string { return e.Err.Error() } + +// Unwrap makes LeaseRenewalRefusal match the refusal it carries. +func (e *LeaseRenewalRefusal) Unwrap() error { return e.Err } + +// LeaseRenewalRefusalCommits reports whether err is a RenewLeaseInTx refusal +// whose transaction a legacy HeartbeatIssue leg must COMMIT rather than roll +// back: one that read the issue row (Found), the only refusal that can carry a +// write, the orphaned lease row it deletes. A refusal whose issue read failed +// deletes nothing, and may follow a fast-path UPDATE that already extended an +// orphan row's expiry, so its transaction rolls back — the rule the role's +// HeartbeatRefusalCommits applies to the same refusal. +func LeaseRenewalRefusalCommits(err error) bool { + var refused *LeaseRenewalRefusal + return errors.As(err, &refused) && refused.Found +} + +// RenewLeaseInTx is the body of HeartbeatIssueInTx, reporting the renewal: +// see that function for what it writes and, above all, what it must never +// write. Its SQL text is the heartbeat's own and must stay so: the Postgres +// dialect rewrites the one IF() in it keyed on its `update leases set` +// prefix. +// +//nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) +func RenewLeaseInTx(ctx context.Context, tx DBTX, id, actor string) (LeaseRenewal, error) { + // THE BOUND CLOCK IS UNROUNDED; THE REPORTED ONE IS ROUNDED. The lease + // columns are DATETIME (whole seconds), which round a sub-second value on + // store, so the result reports now rounded to the second: exactly what + // the row now holds. The binds stay unrounded on purpose: Dolt counts a + // value-identical UPDATE as 0 rows affected even under CLIENT_FOUND_ROWS, + // and binding the rounded value would make every same-second refresh look + // like a miss and take the slow path (several extra reads on the hottest + // write in the fleet). now := time.Now().UTC() + reported := now.Round(time.Second) + ttl := leaseTTL(ctx) // granted_node is backfilled, never overwritten: a lease row whose // provenance is unknown (granted before ignored migration 0016, or before // this replica was named) is being kept alive through THIS node's store, @@ -340,28 +448,69 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { // `bd unclaim --force ` for a single stranded unit once that replica is // gone for good. // - // SCOPE, precisely: the no-re-home invariant holds on THIS fast path only. - // The disambiguation fallback below re-homes, by design and unavoidably — - // when `holder = ?` misses because the lease row spells the same identity - // differently (ga-v2k49/ga-wzl83), it re-arms through UpsertLeaseInTx, whose - // ON DUPLICATE KEY UPDATE stamps the LOCAL node over whatever was there. - // So "a heartbeat never re-homes a foreign lease" is false for a - // spelling-mismatched holder, and RestoreLeaseOnImportInTx can relabel in - // the other direction (it takes granted_node from the snapshot when the - // local row is expired). Neither is changed here: both are pre-existing - // paths with their own rationale, and the point of this note is that the - // invariant is a property of the common path, not of the lease row. + // SCOPE, precisely: the no-re-home invariant holds on every REFRESH — this + // fast path and the slow path's in-place refresh of a row the caller + // already holds (same second, or the same identity spelled differently, + // ga-v2k49/ga-wzl83), which keeps granted_at and granted_node. Only a + // RE-ARM re-homes: when the caller holds a live claim and no lease row the + // caller holds exists, UpsertLeaseInTx mints one and stamps the LOCAL node. + // RestoreLeaseOnImportInTx can relabel in the other direction (it takes + // granted_node from the snapshot when the local row is expired); that is a + // pre-existing path with its own rationale. + // + // THE LIVE-CLAIM PREDICATE. The fast path refreshes the caller's row only + // while the issue is still in_progress under the caller (verbatim, the + // holder predicate's own rule). "A lease row exists only while its claim + // is live" is an invariant every claim-ending path maintains by deleting + // the row, and the slow path's re-arm keeps it against a concurrent ender + // by reminting row_lock (see below). The predicate is the second line: a + // lease row orphaned any other way (a binary without that remint, a + // hand-edited table) fails the fast path, and the slow path refuses AND + // deletes it, instead of every later heartbeat refreshing it and answering + // success for a claim that no longer exists. The `update leases set` + // prefix and the one IF() the Postgres dialect rewrites are unchanged. + // + // THE COMPARISON IS VERBATIM (assignee = the holder as sent), deliberately, + // as the holder predicate is: a holder whose lease row and assignee are + // spelled differently (ga-v2k49's separator case) misses here on every + // beat and takes the slow path, which judges under actorMatches and + // refreshes the row in place. Correct, a few reads slower, and only for + // that spelling mismatch; the common case pays one indexed count read. + // + // FOR A POSTGRES PORT: the refresh now READS the issues row. Under + // SERIALIZABLE that is a read-write dependency on every concurrent write + // to the same issue (the holder's own edits racing its heartbeat), so + // expect more serialization failures on this statement, and keep it inside + // the retrying write transaction. result, err := tx.ExecContext(ctx, ` UPDATE leases SET lease_expires_at = ?, heartbeat_at = ?, granted_node = IF(COALESCE(granted_node, '') = '', ?, granted_node) WHERE issue_id = ? AND holder = ? - `, now.Add(leaseTTL(ctx)), now, NodeID(ctx), id, actor) + `, now.Add(ttl), now, NodeID(ctx), id, actor) if err != nil { - return fmt.Errorf("failed to heartbeat issue: %w", err) + return LeaseRenewal{}, fmt.Errorf("failed to heartbeat issue: %w", err) } rows, err := result.RowsAffected() if err != nil { - return fmt.Errorf("failed to get rows affected: %w", err) + return LeaseRenewal{}, fmt.Errorf("failed to get rows affected: %w", err) + } + if rows > 0 { + // The live-claim check, in the same transaction as the UPDATE (so on + // the same snapshot, and atomically with it: a refusal below commits + // nothing the caller keeps but the orphan delete). It is a separate + // read rather than an EXISTS in the UPDATE because Dolt reports a + // value-identical UPDATE that carries a subquery as 0 rows affected + // even under CLIENT_FOUND_ROWS, which would send every same-second + // refresh down the slow path. + var live int + if err := tx.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issues WHERE id = ? AND status = 'in_progress' AND assignee = ?`, id, actor, + ).Scan(&live); err != nil { + return LeaseRenewal{}, fmt.Errorf("failed to heartbeat issue: %w", err) + } + if live == 0 { + rows = 0 // an orphan or a spelling mismatch: the slow path decides + } } if rows == 0 { // No lease row. Disambiguate from the issue row: gone @@ -374,7 +523,47 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { fmt.Sprintf("SELECT COALESCE(assignee, ''), status FROM %s WHERE id = ?", issueTable), id, ).Scan(&assignee, &status) if qerr != nil { - return fmt.Errorf("%w: %s", storage.ErrNotClaimable, id) + return LeaseRenewal{}, &LeaseRenewalRefusal{ + IssueID: id, ReadErr: qerr, Wisp: isWisp, + Err: fmt.Errorf("%w: %s", storage.ErrNotClaimable, id), + } + } + // A refusal deletes the caller's own lease row, if one survives on an + // issue the caller no longer holds a live claim on: that row is an + // orphan (the re-arm race above), and leaving it would keep a stale + // lease on an issue that is open, closed or someone else's. The holder + // fence keeps it to the caller's own row; a live holder's lease is + // never touched. The fence is judged under actorMatches, as the + // in-place refresh below judges it: a row held under another spelling + // of the caller (ga-v2k49) is the caller's orphan too, so its holder + // is read and bound back, the refresh's own lookup, on this cold path + // only. The delete is judged in this transaction's snapshot, which is + // the snapshot that just found no live claim for the caller. + refused := func(err error) error { + refusal := &LeaseRenewalRefusal{ + IssueID: id, Found: true, Wisp: isWisp, Assignee: assignee, Status: status, Err: err, + } + if isWisp { + return refusal + } + var leaseHolder string + lerr := tx.QueryRowContext(ctx, `SELECT holder FROM leases WHERE issue_id = ?`, id).Scan(&leaseHolder) + switch { + case errors.Is(lerr, sql.ErrNoRows): + return refusal + case lerr != nil: + return fmt.Errorf("read lease for %s: %w", id, lerr) + case !actorMatches(leaseHolder, actor): + return refusal + } + res, derr := tx.ExecContext(ctx, `DELETE FROM leases WHERE issue_id = ? AND holder = ?`, id, leaseHolder) + if derr != nil { + return fmt.Errorf("delete orphaned lease for %s: %w", id, derr) + } + if n, nerr := res.RowsAffected(); nerr == nil && n > 0 { + refusal.OrphanDeleted = true + } + return refusal } // Judged under actorMatches, not verbatim (ga-v2k49): the primary // UPDATE's own `holder = ?` predicate above stays a verbatim SQL @@ -383,11 +572,11 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { // above) just to bind back its own holder value. A spelling // difference across layers (ga-wzl83) already makes that predicate // affect 0 rows and fall here; fixing the disambiguation instead - // means the caller still self-heals correctly (re-arms below, under - // its current spelling) at the cost of the slow path on a spelling - // mismatch, never on the byte-identical common case. + // means the caller still self-heals correctly (its row is refreshed in + // place below, under its current spelling) at the cost of the slow + // path on a spelling mismatch, never on the byte-identical common case. if assignee != "" && !actorMatches(assignee, actor) { - return fmt.Errorf("%w by %s", storage.ErrAlreadyClaimed, assignee) + return LeaseRenewal{}, refused(fmt.Errorf("%w by %s", storage.ErrAlreadyClaimed, assignee)) } if !isWisp && actorMatches(assignee, actor) && status == string(types.StatusInProgress) { // The caller genuinely holds the claim but has no lease row — e.g. @@ -396,13 +585,80 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { // semantics, or (ga-v2k49) the existing lease row's holder is a // different spelling of the same identity (ga-wzl83) and the // primary UPDATE's verbatim predicate above missed it. A real - // worker's heartbeat re-arms recovery — under actor's current - // spelling, which is what the next heartbeat's fast path will see. - return UpsertLeaseInTx(ctx, tx, id, actor, now, leaseTTL(ctx)) + // worker's heartbeat recovers under actor's current spelling: a + // lease row the caller already holds is refreshed in place (its + // holder rewritten to that spelling, which the next heartbeat's + // UPDATE matches), and a missing one is re-armed. An ASSIGNEE + // spelled differently from actor still fails the verbatim + // live-claim check, so that holder takes this path on every beat. + // + // FIRST, is there a lease row for this issue the caller already + // holds? Then this is a refresh the fast path missed without + // anything being wrong: on embedded Dolt an UPDATE that changes no + // value (a second heartbeat in the same second) reports 0 rows, and + // a holder spelled differently from the assignee misses the + // verbatim live-claim predicate. Refresh that row in place, keeping + // its granted_at and granted_node (the fast path's no-re-home rule, + // with the same backfill), and report no re-arm. + var leaseHolder string + lerr := tx.QueryRowContext(ctx, `SELECT holder FROM leases WHERE issue_id = ?`, id).Scan(&leaseHolder) + switch { + case lerr == nil && actorMatches(leaseHolder, actor): + if _, err := tx.ExecContext(ctx, ` + UPDATE leases SET lease_expires_at = ?, heartbeat_at = ?, holder = ?, + granted_node = IF(COALESCE(granted_node, '') = '', ?, granted_node) + WHERE issue_id = ? AND holder = ? + `, now.Add(ttl), now, actor, NodeID(ctx), id, leaseHolder); err != nil { + return LeaseRenewal{}, fmt.Errorf("failed to heartbeat issue: %w", err) + } + return LeaseRenewal{HeartbeatAt: reported, ExpiresAt: reported.Add(ttl)}, nil + case lerr != nil && !errors.Is(lerr, sql.ErrNoRows): + return LeaseRenewal{}, fmt.Errorf("read lease for %s: %w", id, lerr) + } + // No lease row the caller holds: grant one. THIS IS A MUTATION, + // recorded as every bead mutator records one: row_lock reminted, + // updated_at stamped, a lease_rearmed event, a journal entry and a + // version row (the legs then make the version-control commit and + // fire the update hook, keyed on LeaseRenewal.Rearmed). Why the + // remint is load-bearing: The live claim this branch acts on was read in + // THIS transaction's snapshot, but a snapshot cannot see a release + // or close committing concurrently, and a lease INSERT shares no + // row with such an ender (its DELETE FROM leases matched nothing in + // its own snapshot): Dolt would commit both and leave an orphaned + // lease on an open or closed issue. Every claim-ending path + // rewrites row_lock (the freshRowLock invariant), so reminting it + // here makes the grant COLLIDE with any concurrent ender, and the + // loser is replayed against the winner's state: either the ender + // sees and deletes this lease, or this heartbeat sees the ended + // claim and is refused. Concurrent re-arms collide with each other + // the same way, so exactly one reports Rearmed. + // + // THE COST, accepted: a re-arm (a hand-doled claim's first + // heartbeat, once) remints the holder's revision token, so a + // guarded write composed before it fails once and re-reads. A + // REFRESH never touches the issue row and records nothing. + if _, err := tx.ExecContext(ctx, `UPDATE issues SET row_lock = ?, updated_at = ? WHERE id = ?`, + freshRowLock(), now, id); err != nil { + return LeaseRenewal{}, fmt.Errorf("remint row_lock for %s: %w", id, err) + } + if err := UpsertLeaseInTx(ctx, tx, id, actor, now, ttl); err != nil { + return LeaseRenewal{}, err + } + if err := RecordFullEventInTable(ctx, tx, "events", id, types.EventLeaseRearmed, actor, "", actor); err != nil { + return LeaseRenewal{}, fmt.Errorf("record lease re-arm event for %s: %w", id, err) + } + if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { + return LeaseRenewal{}, err + } + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return LeaseRenewal{}, err + } + return LeaseRenewal{HeartbeatAt: reported, ExpiresAt: reported.Add(ttl), Rearmed: true}, nil } - return fmt.Errorf("%w: %s status %s", storage.ErrNotClaimable, id, status) + return LeaseRenewal{}, refused(fmt.Errorf("%w: %s status %s", storage.ErrNotClaimable, id, status)) } - // DELIBERATELY NOT JOURNALED — do not add an emit here. A heartbeat is a + // A REFRESH IS DELIBERATELY NOT JOURNALED — do not add an emit here (the + // re-arm above is the one heartbeat path recorded as a mutation). A heartbeat is a // high-frequency lease keepalive: it writes only the clone-local `leases` // table (lease-liveness state), never a durable bead field. Journaling it // would put a full-snapshot write plus the shared seq-counter serialization @@ -410,8 +666,8 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { // lease state lives on the working-set plane and expires on its own. Lease // RECLAIM is the opposite case and does journal: it clears assignee and // reverts status, which is durable bead state. The decision is pinned by - // journalExemptMutations in journal_completeness_test.go. - return nil + // mutationEntryPoints' heartbeat note in journal_completeness_test.go. + return LeaseRenewal{HeartbeatAt: reported, ExpiresAt: reported.Add(ttl)}, nil } // warnReplica writes a replica-guard audit line to STDERR (never stdout — diff --git a/internal/storage/issueops/row_lock_guard_test.go b/internal/storage/issueops/row_lock_guard_test.go index e43565398..077269263 100644 --- a/internal/storage/issueops/row_lock_guard_test.go +++ b/internal/storage/issueops/row_lock_guard_test.go @@ -68,8 +68,9 @@ var sqlWriteKeywordRe = regexp.MustCompile(`(?i)\b(?:SET|VALUES|SELECT)\b`) // the token would clobber a concurrent whole-row CAS (ExpectedVersion) for // no content change. Analysis absorbed from gastownhall/beads#4682. // (The PR's other exemption, the lease heartbeat, is moot on main: since -// bd-lrgn1 heartbeats live entirely in the ephemeral leases table and never -// touch the issues row.) +// bd-lrgn1 a heartbeat refresh lives entirely in the ephemeral leases +// table and never touches the issues row, and a re-arm remints row_lock on +// purpose; see the freshRowLock invariant.) // - "compaction_level = ": compaction apply/restore rewrites bookkeeping and // compacted body text only — cells the reclaim/close races don't care // about, exempted by name in the freshRowLock invariant. The assignment diff --git a/internal/storage/issueops/version_completeness_test.go b/internal/storage/issueops/version_completeness_test.go index 611c55c6d..ae3448df4 100644 --- a/internal/storage/issueops/version_completeness_test.go +++ b/internal/storage/issueops/version_completeness_test.go @@ -89,6 +89,12 @@ var versionCompositeMutations = []string{ // does. Every write plumbing bottoms out in one of these; the structural // cross-check (TestEveryBeadMutatorMintsOrIsExempt) keeps the list complete. var versionedEntryPoints = []string{ + // the heartbeat family mints on its RE-ARM path only (recorded as a + // mutation); a refresh writes only the clone-local leases table and mints + // nothing. + "RenewLeaseInTx", + "HeartbeatIssueInTx", + "ExecuteHeartbeatInTx", // create — the singular path mints in place; the batch path defers the // mint past PersistDependenciesWithOptionsResult so the first version // carries the creation-time edge set. @@ -244,7 +250,6 @@ var versionNeverMints = map[string]string{ "UpdateIssueIDInTx": "rename: out of contract by design", "ApplyCompactionInTx": "compaction bookkeeping: out of contract by design", "RestoreFromSnapshotInTx": "restore: Phase 3", - "HeartbeatIssueInTx": "lease keepalive: writes only the clone-local leases table, never a durable bead field", "BootstrapInTx": "bootstrap: config and metadata tables only, no issue-plane row", } diff --git a/internal/storage/issueops/version_history.go b/internal/storage/issueops/version_history.go index ad2fb66c1..ff579fbd6 100644 --- a/internal/storage/issueops/version_history.go +++ b/internal/storage/issueops/version_history.go @@ -98,6 +98,14 @@ func versionedHistoryEnabled(tx DBTX) bool { return on } +// VersionedHistoryEnabledFor reports whether versioned history is active on +// tx, the transaction ScopeVersionedHistoryTransaction bound. A commit path +// that stages a NAMED set of tables uses it to add +// VersionedHistoryStagedTables only when the transaction could have minted. +func VersionedHistoryEnabledFor(tx DBTX) bool { + return versionedHistoryEnabled(tx) +} + // VersionedHistoryStagedTables names every table RecordVersionInTx writes, so // a staging path can add them to whatever it was already going to stage. // diff --git a/internal/storage/role_accessor_decorator_test.go b/internal/storage/role_accessor_decorator_test.go index 540518354..62aea5354 100644 --- a/internal/storage/role_accessor_decorator_test.go +++ b/internal/storage/role_accessor_decorator_test.go @@ -68,11 +68,11 @@ func roleAccessorNamesOf(surface reflect.Type) (names, unclassified []string) { } // TestEveryStoreRoleAccessorIsClassified fails when DoltStorage hands out an -// interface the census cannot place. Every one of the twenty-nine today is a +// interface the census cannot place. Every one of the thirty today is a // facade role, so this costs nothing and closes the path where a role surface // grows a package and the census quietly stops covering it. (roleAccessorNames // is derived, so the count is prose and only this sentence goes stale — but it -// does go stale, and the last three sweeps found it saying twenty-eight.) +// does go stale, and the last four sweeps found it behind.) func TestEveryStoreRoleAccessorIsClassified(t *testing.T) { _, unclassified := roleAccessorNamesOf(reflect.TypeOf((*DoltStorage)(nil)).Elem()) for _, accessor := range unclassified { @@ -151,7 +151,7 @@ func assertRoleAccessorsAreDeclared(t *testing.T, decorator reflect.Type) { } // roleAccessorStore is a DoltStorage whose only real methods are the -// twenty-nine role accessors, each answering with a distinguishable sentinel +// thirty role accessors, each answering with a distinguishable sentinel // so a test can tell a decorated surface from a passed-through one. type roleAccessorStore struct { DoltStorage @@ -184,6 +184,7 @@ type roleAccessorStore struct { metadataCAS issueops.MetadataCAS releaser issueops.Releaser leaseReclaimer issueops.LeaseReclaimer + heartbeater issueops.Heartbeater err error } @@ -218,6 +219,7 @@ func newRoleAccessorStore() *roleAccessorStore { metadataCAS: sentinel, releaser: sentinel, leaseReclaimer: sentinel, + heartbeater: sentinel, } } @@ -291,8 +293,11 @@ func (s *roleAccessorStore) Releaser() (issueops.Releaser, error) { func (s *roleAccessorStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return s.leaseReclaimer, s.err } +func (s *roleAccessorStore) Heartbeater() (issueops.Heartbeater, error) { + return s.heartbeater, s.err +} -// roleAccessorSentinel implements twenty-eight of the twenty-nine roles at +// roleAccessorSentinel implements twenty-nine of the thirty roles at // once — every one but memoryops.Memories, whose List collides with // issueops.Reader.List and needs the second sentinel below. // Nothing calls its methods; identity is the whole point. @@ -420,6 +425,9 @@ func (*roleAccessorSentinel) Release(context.Context, issueops.ReleaseRequest) ( func (*roleAccessorSentinel) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { return issueops.ReclaimResult{}, nil } +func (*roleAccessorSentinel) Heartbeat(context.Context, issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + return issueops.HeartbeatResult{}, nil +} // memoryRoleSentinel is the memory role's sentinel, and the one role // memoryRoleSentinel is the remaining role's sentinel, and the one role @@ -455,7 +463,7 @@ func (*memoryRoleSentinel) List(context.Context, memoryops.ListRequest) (memoryo // ReadyCounter, Querier and InitVerifier deliberately return the inner surface // unwrapped, // each in its own hook_*.go. The ones in that column that are NOT reads are -// WorkspaceConfig, Memories, VersionReconciler, Bootstrapper, Sweeper and +// WorkspaceConfig, Memories, VersionReconciler, Bootstrapper, Sweeper, // Deleter: a settings write changes the workspace rather than a bead, a // remembered insight names no bead and there is no on_remember to fire, a // version marker names no bead at all, a bootstrap lands on a workspace whose @@ -502,6 +510,7 @@ func TestHookFiringStoreWrapsTheWriteRolesAndPassesTheReadsThrough(t *testing.T) {"Deleter", func() (any, error) { return store.Deleter() }, inner.deleter, false}, {"Bootstrapper", func() (any, error) { return store.Bootstrapper() }, inner.bootstrapper, false}, {"InitVerifier", func() (any, error) { return store.InitVerifier() }, inner.verifier, false}, + {"Heartbeater", func() (any, error) { return store.Heartbeater() }, inner.heartbeater, true}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() @@ -561,6 +570,7 @@ func TestHookFiringStoreRoleAccessorsPropagateInnerErrors(t *testing.T) { {"Deleter", func() (any, error) { return store.Deleter() }}, {"Bootstrapper", func() (any, error) { return store.Bootstrapper() }}, {"InitVerifier", func() (any, error) { return store.InitVerifier() }}, + {"Heartbeater", func() (any, error) { return store.Heartbeater() }}, } { t.Run(test.name, func(t *testing.T) { if _, err := test.got(); !errors.Is(err, want) { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index eaff2090c..11bdd0033 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -387,6 +387,16 @@ type Storage interface { // assignee and status per reverted row, which is on_update, the same // event Releaser fires. See hook_lease_reclaimer.go. LeaseReclaimer() (issueops.LeaseReclaimer, error) + // Heartbeater returns the lease-refresh surface for this store: push the + // lease the caller holds on one issue forward by the store's TTL. + // + // It is a WRITE role and its hook decorator WRAPS: a refresh writes only + // the ephemeral lease row and fires nothing, but a re-arm (a live claim + // with no lease row, HeartbeatResult.Rearmed) is recorded as a mutation + // and fires on_update once. That is why RoleFiresHooks answers true for + // it, and a server must take it off a peeled store. See + // hook_heartbeater.go. + Heartbeater() (issueops.Heartbeater, error) // MoleculeStepper returns the advance-a-molecule surface for this store: // find a molecule's next ready step after one closed and, on request, diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index ac268f448..72bd10f73 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "events_journal.go", "external_doltserver_provider.go", "graph_counter.go", + "heartbeater.go", "importer.go", "init_verifier.go", "issue_claimer.go", @@ -120,6 +121,7 @@ go_test( "expected_revision_contract_test.go", "external_doltserver_provider_test.go", "graph_counter_contract_test.go", + "heartbeater_contract_test.go", "importer_contract_test.go", "importer_uow_test.go", "issue_claimer_test.go", diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index 4fd396e7a..beebf330b 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -5,9 +5,16 @@ import ( "database/sql" "database/sql/driver" "errors" + "fmt" + "log" + "slices" + "time" + + "github.com/cenkalti/backoff/v4" "github.com/steveyegge/beads/internal/storage/domain/db" "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/schema" ) type doltServerTx struct { @@ -80,6 +87,156 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { return t.closeOpenTxAfterFailure(ctx, err) } +const ( + // scopedCommitMaxElapsed bounds the retries of CommitScoped's Dolt commit. + // It is short for the reason the DoltStore's postTxCommitMaxElapsed is: + // the failure is logged rather than returned, so a long fight for a commit + // the next one on the branch carries anyway buys nothing. + scopedCommitMaxElapsed = 5 * time.Second + // scopedCommitGrace pads the detached context past the retry budget. The + // budget only stops scheduling attempts; the deadline is what cancels one + // already hung in flight. + scopedCommitGrace = 2 * time.Second +) + +// CommitScoped commits the transaction and then makes a Dolt commit of +// exactly tables (plus the versioned-history tables when this transaction +// could have minted) named message: DOLT_ADD per table, then +// DOLT_COMMIT('-m') — never '-Am', so another writer's uncommitted change +// elsewhere in the working set is neither committed nor attributed to this +// message. With nothing staged, or under a deferred version commit +// (dolt.auto-commit batch/off), there is no Dolt commit. +// +// THE DOLT COMMIT RUNS AFTER THE SQL COMMIT, never inside the transaction: +// the ordering contract of the DoltStore funnel (runIssueOperationTxWithMessage, +// LatentLabsSpace/NEXUS#92). DOLT_ADD stages whole tables from the session's +// transaction root, so staging inside the open transaction builds the Dolt +// commit from its BEGIN-time snapshot and can write rows a concurrent writer +// committed in the meantime back to their old values. Committed first, this +// transaction's writes go through Dolt's commit-time merge with those +// writers, and the DOLT_ADD on the same pinned session, now outside any +// transaction, stages the merged state. +// +// A serialization failure of the SQL COMMIT is returned for the caller to redo +// the whole unit of work, as Commit's is. Once the SQL COMMIT succeeds the +// write is durable, so the Dolt commit after it never fails CommitScoped (see +// commitStagedTables). +func (t *doltServerTx) CommitScoped(ctx context.Context, tables []string, message string) error { + if t.done { + return errors.New("uow: commit: already done") + } + if message == "" || len(tables) == 0 || issueops.VersionCommitDeferred(ctx) { + return t.Commit(ctx, "") + } + // Resolved before the commit, while the versioned-history scope is bound + // to the pinned session. + staged := append([]string(nil), tables...) + if issueops.VersionedHistoryEnabledFor(t.conn) { + for _, table := range issueops.VersionedHistoryStagedTables() { + if !slices.Contains(staged, table) { + staged = append(staged, table) + } + } + } + if _, err := t.conn.ExecContext(ctx, "COMMIT;"); err != nil { + if isSerializationError(err) { + return err + } + return t.closeOpenTxAfterFailure(ctx, err) + } + t.done = true + clean := t.commitStagedTables(ctx, staged, message) + if t.recheckBlockedAfterCommit(ctx, issueops.TakeBlockedRecheck(t.conn)) && clean { + t.releaseConn() + } else { + t.poisonConn() + } + return nil +} + +// commitStagedTables is CommitScoped's Dolt commit, made on the pinned session +// after the transaction committed. It runs detached from the caller's +// cancellation on its own short budget, so a deadline landing after the SQL +// COMMIT does not skip it, and replays only the conflicts that leave nothing +// behind (serialization failures and Dolt's @@autocommit rollback, which is +// how a conflict surfaces outside a transaction), so re-staging and +// committing again is safe. That is doltAddAndCommitPostTx's retry. +// +// A failure is logged, never returned: the write is durable, and an error +// would make RunTxScoped replay a committed unit of work, which for a +// heartbeat re-arm then reports a refresh. The change rides the next Dolt +// commit on the branch, as a failed post-transaction commit does in the +// DoltStore funnel. Unlike the funnel, this leg counts nothing: uow records +// no metrics, so bd.db.post_tx_commit_dropped, the funnel's counter for the +// same drop, never sees one made here, and the log line is its only signal. +// It reports whether the session may go back to the pool; after a failure its +// state is unknown (a statement cut off mid-flight, or under @@autocommit=0 +// the implicit transaction a DOLT_ADD opened), so it may not. +func (t *doltServerTx) commitStagedTables(ctx context.Context, tables []string, message string) bool { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), scopedCommitMaxElapsed+scopedCommitGrace) + defer cancel() + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = txRetryInitialInterval + bo.MaxElapsedTime = scopedCommitMaxElapsed + var lastErr error + err := backoff.Retry(func() error { + err := t.stageAndCommitOnce(ctx, tables, message) + if err == nil { + return nil + } + lastErr = err + if isSerializationError(err) || isDoltAutocommitRollbackError(err) { + return err + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) + if err != nil && lastErr != nil && !errors.Is(err, lastErr) { + // The deadline expiring between attempts returns the bare context + // error; keep the last real failure, the only diagnostic of a dropped + // commit. + err = fmt.Errorf("%w (last attempt: %w)", err, lastErr) + } + if err != nil { + log.Printf("warning: uow: dolt commit %q failed after its SQL commit (data already committed; change rides the next dolt commit): %v", message, err) + return false + } + return true +} + +// stageAndCommitOnce is one attempt at commitStagedTables: stage each table, +// then commit the staged set alone. +func (t *doltServerTx) stageAndCommitOnce(ctx context.Context, tables []string, message string) error { + for _, table := range tables { + if err := schema.DrainCall(ctx, t.conn, "CALL DOLT_ADD(?)", table); err != nil { + return fmt.Errorf("dolt add %s: %w", table, err) + } + } + // Sessions on one branch share the working set, so a concurrent writer's + // Dolt commit can already have taken this transaction's rows under its + // own message, leaving nothing staged here: skip the empty commit Dolt + // would refuse. The same race between this check and the commit surfaces + // as "nothing to commit", which here means exactly that and not a lost + // write (the SQL COMMIT has landed), so it is no failure either; it is + // logged, as the DoltStore funnel logs it, so the missing audit line is + // explicable. + staged, err := issueops.HasStagedChanges(ctx, t.conn) + if err != nil { + return err + } + if !staged { + return nil + } + err = schema.DrainCall(ctx, t.conn, "CALL DOLT_COMMIT('-m', ?)", message) + if err != nil { + if !issueops.IsNothingToCommitError(err) { + return fmt.Errorf("dolt commit: %w", err) + } + log.Printf("uow: dolt commit %q made no dolt commit (nothing to commit): the staged change was most likely absorbed into a concurrent writer's commit (data already committed)", message) + } + return nil +} + // commitStatement picks the statement that ends the open transaction on the // pinned session for a commit named message. func (t *doltServerTx) commitStatement(ctx context.Context, message string) (string, []interface{}, error) { @@ -95,12 +252,15 @@ func (t *doltServerTx) commitStatement(ctx context.Context, message string) (str } // An empty message selects the EPHEMERAL commit form (bd-aq0ql): a plain // SQL COMMIT persists the transaction's writes into the working set - // without minting a Dolt commit or history. It has two entry points: + // without minting a Dolt commit or history. It has these entry points: // - // - uow.RunTxEphemeral, for work that touches ONLY dolt_ignored state — - // today the leases table (bd-lrgn1), whose heartbeats must never create - // commits. RunTx/RunTxResult treat an empty commitMsg as "nothing to - // commit" and never call Commit at all, so they cannot reach it that way. + // - uow.RunTxEphemeral, for work that touches ONLY dolt_ignored state. + // RunTx/RunTxResult treat an empty commitMsg as "nothing to commit" and + // never call Commit at all, so they cannot reach it that way. + // - uow.RunTxScoped, for an attempt that names no tables — a lease + // refresh (bd-lrgn1), which writes only the dolt_ignored leases table + // and must never create a commit. A heartbeat that re-arms names its + // tables instead and takes the scoped commit. // - RunTx/RunTxResult under a deferred-version-commit context, via the // blanking just above (GH#4995). There the writes are ordinary versioned // rows and the ONLY-dolt_ignored-state rationale does not apply: the diff --git a/internal/storage/uow/doltserver_tx_test.go b/internal/storage/uow/doltserver_tx_test.go index 036291ba8..b665ea1dc 100644 --- a/internal/storage/uow/doltserver_tx_test.go +++ b/internal/storage/uow/doltserver_tx_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + mysql "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -285,3 +286,134 @@ func TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit(t *testing.T) { require.NoError(t, mock.ExpectationsWereMet()) assert.Equal(t, 1, p.db.Stats().OpenConnections) } + +// matchHasStaged matches the staged-set check issueops.HasStagedChanges runs +// before CommitScoped's DOLT_COMMIT('-m'). +const matchHasStaged = `SELECT COUNT\(\*\) FROM dolt_status WHERE staged = 1` + +// expectScopedDoltCommit arms one attempt of CommitScoped's Dolt commit: a +// DOLT_ADD per table, the staged-set check reporting staged rows, and, when +// something is staged, a DOLT_COMMIT('-m') answering commitErr. +func expectScopedDoltCommit(mock sqlmock.Sqlmock, tables []string, staged int, message string, commitErr error) { + for _, table := range tables { + mock.ExpectQuery(`CALL DOLT_ADD\(\?\)`).WithArgs(table). + WillReturnRows(sqlmock.NewRows([]string{"status"}).AddRow(0)) + } + mock.ExpectQuery(matchHasStaged).WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(staged)) + if staged == 0 { + return + } + commit := mock.ExpectQuery(`CALL DOLT_COMMIT\('-m', \?\)`).WithArgs(message) + if commitErr != nil { + commit.WillReturnError(commitErr) + } else { + commit.WillReturnRows(sqlmock.NewRows([]string{"hash"}).AddRow("c0ffee")) + } +} + +// TestDoltServerTxCommitScopedCommitsSQLBeforeDoltCommit pins the +// LatentLabsSpace/NEXUS#92 ordering on the uow leg: CommitScoped ends the SQL +// transaction with a plain COMMIT and only then stages and Dolt-commits its +// tables, on the same pinned session. Staged inside the open transaction, +// DOLT_ADD builds the Dolt commit from the transaction's start snapshot and +// writes a concurrent writer's committed rows back to their old values. +func TestDoltServerTxCommitScopedCommitsSQLBeforeDoltCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + expectScopedDoltCommit(mock, []string{"events", "issues"}, 2, "bd: re-arm lease x", nil) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + require.NoError(t, tx.(ScopedCommitter).CommitScoped(context.Background(), []string{"events", "issues"}, "bd: re-arm lease x")) + require.NoError(t, mock.ExpectationsWereMet(), "the SQL COMMIT must come first, then DOLT_ADD per table and DOLT_COMMIT('-m')") + assert.Equal(t, 1, p.db.Stats().OpenConnections, "a cleanly committed session may return to the pool") + require.ErrorContains(t, tx.Commit(context.Background(), ""), "already done") +} + +// TestDoltServerTxCommitScopedSwallowsAFailedDoltCommit: once the SQL COMMIT +// has landed the write is durable, so a Dolt commit that fails after it must +// not fail CommitScoped (RunTxScoped would replay a committed unit of work). +// The session, whose state the failed statement leaves unknown, is discarded. +func TestDoltServerTxCommitScopedSwallowsAFailedDoltCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(`CALL DOLT_ADD\(\?\)`).WithArgs("issues").WillReturnError(errors.New("dolt add exploded")) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + require.NoError(t, tx.(ScopedCommitter).CommitScoped(context.Background(), []string{"issues"}, "bd: re-arm lease x"), + "a committed write must not be reported as failed") + require.NoError(t, mock.ExpectationsWereMet(), "a failure other than a conflict is not retried") + assert.Equal(t, 0, p.db.Stats().OpenConnections, "the session must be discarded, not pooled") + require.ErrorContains(t, tx.Commit(context.Background(), ""), "already done") +} + +// TestDoltServerTxCommitScopedRetriesAConflictedDoltCommit: outside a +// transaction, a Dolt commit that conflicts with a concurrent writer is rolled +// back as an @@autocommit merge conflict (1105), not 1213, and is retried like +// one. +func TestDoltServerTxCommitScopedRetriesAConflictedDoltCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + conflict := &mysql.MySQLError{Number: 1105, Message: "Merge conflict detected, @autocommit transaction rolled back"} + expectScopedDoltCommit(mock, []string{"issues"}, 1, "bd: re-arm lease x", conflict) + expectScopedDoltCommit(mock, []string{"issues"}, 1, "bd: re-arm lease x", nil) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + require.NoError(t, tx.(ScopedCommitter).CommitScoped(context.Background(), []string{"issues"}, "bd: re-arm lease x")) + require.NoError(t, mock.ExpectationsWereMet(), "the conflicted attempt must be staged and committed again") + assert.Equal(t, 1, p.db.Stats().OpenConnections, "a session whose retry committed may return to the pool") +} + +// TestDoltServerTxCommitScopedReturnsASerializationFailure: a serialization +// failure of the SQL COMMIT is returned for RunTxScoped to redo the whole unit +// of work, and no Dolt commit follows a COMMIT that did not land. +func TestDoltServerTxCommitScopedReturnsASerializationFailure(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnError(&mysql.MySQLError{Number: 1213, Message: "serialization failure"}) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + err = tx.(ScopedCommitter).CommitScoped(context.Background(), []string{"issues"}, "bd: re-arm lease x") + require.True(t, isSerializationError(err), "err = %v, want the serialization failure", err) + require.NoError(t, mock.ExpectationsWereMet(), "nothing may be staged or committed after a failed COMMIT") +} + +// TestDoltServerTxCommitScopedMakesNoEmptyDoltCommit: a concurrent writer's +// Dolt commit can take this transaction's rows before its own Dolt commit +// runs, seen either before the commit (nothing staged) or by it ("nothing to +// commit"). Neither is a failure: unlike Commit's in-transaction commit, the +// rows are already committed. +func TestDoltServerTxCommitScopedMakesNoEmptyDoltCommit(t *testing.T) { + for _, tc := range []struct { + name string + staged int + commitErr error + }{ + {name: "nothing staged", staged: 0}, + {name: "nothing to commit", staged: 1, commitErr: errors.New("nothing to commit")}, + } { + t.Run(tc.name, func(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + expectScopedDoltCommit(mock, []string{"issues"}, tc.staged, "bd: re-arm lease x", tc.commitErr) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + require.NoError(t, tx.(ScopedCommitter).CommitScoped(context.Background(), []string{"issues"}, "bd: re-arm lease x")) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections, "the session is clean and may return to the pool") + }) + } +} diff --git a/internal/storage/uow/errors.go b/internal/storage/uow/errors.go index 217d5904e..b0323a42d 100644 --- a/internal/storage/uow/errors.go +++ b/internal/storage/uow/errors.go @@ -33,6 +33,21 @@ func isSerializationError(err error) bool { return false } +// isDoltAutocommitRollbackError reports whether err is Dolt rolling back a +// statement it ran under @@autocommit because the statement's commit-time +// merge conflicted with a concurrent writer (MySQL 1105, "Merge conflict +// detected, @autocommit transaction rolled back"). Outside an explicit +// transaction this, not 1213, is how such a conflict surfaces, and like a +// serialization failure it guarantees the statement left nothing behind. It +// mirrors the DoltStore's classifier of the same name. +func isDoltAutocommitRollbackError(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && + mysqlErr.Number == 1105 && + (mysqlErr.Message == "Merge conflict detected, @autocommit transaction rolled back" || + strings.HasPrefix(mysqlErr.Message, "Merge conflict detected, @autocommit transaction rolled back.")) +} + // isDatabaseExistsError reports whether err is the server refusing a bare // CREATE DATABASE because the database already exists (MySQL 1007, // ER_DB_CREATE_EXISTS). The message fallback matches how the rest of the diff --git a/internal/storage/uow/errors_test.go b/internal/storage/uow/errors_test.go index 0796af0ac..1a9d8ce77 100644 --- a/internal/storage/uow/errors_test.go +++ b/internal/storage/uow/errors_test.go @@ -59,6 +59,37 @@ func TestIsDatabaseNotFoundError(t *testing.T) { } } +// The DoltStore's table for its twin classifier, plus the prefix boundary: +// commitStagedTables replays exactly what this accepts, so a false positive +// re-runs a Dolt commit that failed for another reason. +func TestIsDoltAutocommitRollbackError(t *testing.T) { + exact := &mysql.MySQLError{ + Number: 1105, + Message: "Merge conflict detected, @autocommit transaction rolled back", + } + + for _, tt := range []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "exact typed Dolt rollback", err: fmt.Errorf("dolt commit: %w", exact), want: true}, + {name: "typed Dolt rollback with recovery guidance", err: &mysql.MySQLError{Number: 1105, Message: exact.Message + ". @autocommit must be disabled"}, want: true}, + {name: "typed text running on past the message", err: &mysql.MySQLError{Number: 1105, Message: exact.Message + "ward"}, want: false}, + {name: "untyped matching text", err: errors.New(exact.Error()), want: false}, + {name: "other typed 1105", err: &mysql.MySQLError{Number: 1105, Message: "Merge conflict detected"}, want: false}, + {name: "typed connection-like 1105", err: &mysql.MySQLError{Number: 1105, Message: "connection lost while validating commit"}, want: false}, + {name: "same text wrong code", err: &mysql.MySQLError{Number: 1213, Message: exact.Message}, want: false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := isDoltAutocommitRollbackError(tt.err); got != tt.want { + t.Fatalf("isDoltAutocommitRollbackError(%v) = %t, want %t", tt.err, got, tt.want) + } + }) + } +} + func TestClassifyUseDatabaseError(t *testing.T) { // The exact server errors observed from dolt 2.1.8 against a credential // granted only one database. diff --git a/internal/storage/uow/heartbeater.go b/internal/storage/uow/heartbeater.go new file mode 100644 index 000000000..75151c829 --- /dev/null +++ b/internal/storage/uow/heartbeater.go @@ -0,0 +1,84 @@ +package uow + +import ( + "context" + "fmt" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" +) + +// HeartbeaterSource is the capability accessor a unit-of-work provider offers +// for the lease-refresh role, the sibling of LeaseReclaimerSource. +type HeartbeaterSource interface { + Heartbeater() (publicops.Heartbeater, error) +} + +// heartbeater refreshes a held lease through a unit of work. +type heartbeater struct { + provider UnitOfWorkProvider +} + +// Heartbeater returns the lease-refresh surface for this provider. +func (p *doltSQLProvider) Heartbeater() (publicops.Heartbeater, error) { + return NewHeartbeater(p) +} + +// NewHeartbeater constructs a public Heartbeater backed by provider. +func NewHeartbeater(provider UnitOfWorkProvider) (publicops.Heartbeater, error) { + if isNilUnitOfWorkProvider(provider) { + return nil, fmt.Errorf("new heartbeater: unit-of-work provider must not be nil") + } + return &heartbeater{provider: provider}, nil +} + +var _ publicops.Heartbeater = (*heartbeater)(nil) + +// Heartbeat reaches domain.IssueUseCase.RenewLease inside ONE unit of work. A +// refresh touches only the dolt_ignored leases table, so it mints no Dolt +// commit and records nothing in dolt_log; a re-arm is recorded as a mutation +// and commits like one. A serialization loser (a heartbeat racing a reclaim, +// release or close) is replayed against a fresh unit of work. +// +// VALIDATION HAPPENS BEFORE THE UNIT OF WORK OPENS, as on the dolt and +// embedded-dolt legs: a refusal then costs no connection. The shared body +// validates again inside the transaction. +func (h *heartbeater) Heartbeat(ctx context.Context, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) { + if err := storageissueops.ValidateHeartbeatRequest(request); err != nil { + return publicops.HeartbeatResult{}, err + } + // A lease refusal commits (it may have deleted an orphaned lease row) and + // is reported after the commit; see HeartbeatRefusalCommits. + // + // THE COMMIT IS SCOPED (RunTxScoped). A refresh or a refusal writes only + // the dolt_ignored leases table and commits with the plain SQL COMMIT: no + // working-set scan, no Dolt commit, and so nothing another writer left + // uncommitted (a batch-mode edit) is swept into a heartbeat commit. A + // re-arm is recorded as a mutation and commits exactly the tables + // LeaseRearmVersionCommit names, never DOLT_COMMIT('-Am'), in a Dolt + // commit made after the SQL COMMIT (ScopedCommitter): built inside the + // transaction, it would write an issues row a concurrent writer changed + // meanwhile back to its value at this heartbeat's start. + type outcome struct { + result publicops.HeartbeatResult + refusal error + } + out, err := RunTxScoped(ctx, h.provider, func(ctx context.Context, uw UnitOfWork) (outcome, []string, string, error) { + result, err := uw.IssueUseCase().RenewLease(ctx, request) + if storageissueops.HeartbeatRefusalCommits(err) { + return outcome{refusal: err}, nil, "", nil + } + if err != nil { + return outcome{}, nil, "", err + } + tables, msg := storageissueops.LeaseRearmVersionCommit(request.IssueID, result.Rearmed) + return outcome{result: result}, tables.Sorted(), msg, nil + }) + if err != nil { + return publicops.HeartbeatResult{}, err + } + if out.refusal != nil { + return publicops.HeartbeatResult{}, out.refusal + } + return out.result, nil +} diff --git a/internal/storage/uow/heartbeater_contract_test.go b/internal/storage/uow/heartbeater_contract_test.go new file mode 100644 index 000000000..6148049d6 --- /dev/null +++ b/internal/storage/uow/heartbeater_contract_test.go @@ -0,0 +1,339 @@ +package uow + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestHeartbeaterContract runs the Heartbeater contract against the +// unit-of-work provider, which reaches internal/storage/issueops. +// ExecuteHeartbeatInTx through domain.IssueUseCase.RenewLease inside one unit +// of work committed with RunTxScoped: the SQL-only form for a refresh, the +// scoped commit for a re-arm. +func TestHeartbeaterContract(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + fixture := newUOWHeartbeaterFixture(t, ctx, "hbc") + t.Run("RefreshesTheHoldersLease", func(t *testing.T) { + conformance.RunHeartbeaterRefreshesTheHoldersLease(t, ctx, fixture) + }) + t.Run("ChangesNothingOnTheRow", func(t *testing.T) { + conformance.RunHeartbeaterChangesNothingOnTheRow(t, ctx, fixture) + }) + t.Run("AGuardedWriteWithThePreHeartbeatRevisionStillWins", func(t *testing.T) { + conformance.RunHeartbeaterAGuardedWriteWithThePreHeartbeatRevisionStillWins(t, ctx, fixture) + }) + t.Run("RefusesAForeignHolder", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAForeignHolder(t, ctx, fixture) + }) + t.Run("RefusesAfterClose", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterClose(t, ctx, fixture) + }) + t.Run("RefusesAfterRelease", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterRelease(t, ctx, fixture) + }) + t.Run("RefusesAfterReclaim", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterReclaim(t, ctx, fixture) + }) + t.Run("RefusesAfterStatusMovedOffInProgress", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAfterStatusMovedOffInProgress(t, ctx, fixture) + }) + t.Run("RefusesAWisp", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAWisp(t, ctx, fixture) + }) + t.Run("RearmsAHandDoledClaim", func(t *testing.T) { + conformance.RunHeartbeaterRearmsAHandDoledClaim(t, ctx, fixture) + }) + t.Run("AnswersNotFoundForAnAbsentID", func(t *testing.T) { + conformance.RunHeartbeaterAnswersNotFoundForAnAbsentID(t, ctx, fixture) + }) + t.Run("RefusesABlankOrOverLongActor", func(t *testing.T) { + conformance.RunHeartbeaterRefusesABlankOrOverLongActor(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheRequest", func(t *testing.T) { + conformance.RunHeartbeaterDoesNotMutateTheRequest(t, ctx, fixture) + }) + t.Run("CommitsOnlyItsOwnTables", func(t *testing.T) { + conformance.RunHeartbeaterCommitsOnlyItsOwnTables(t, ctx, fixture) + }) + t.Run("ASameSecondHeartbeatIsNotARearm", func(t *testing.T) { + conformance.RunHeartbeaterASameSecondHeartbeatIsNotARearm(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAReleasedIssue", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAReleasedIssue(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAClosedIssue", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAClosedIssue(t, ctx, fixture) + }) + t.Run("RefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds", func(t *testing.T) { + conformance.RunHeartbeaterRefusesAnOrphanedLeaseOnAnIssueSomeoneElseHolds(t, ctx, fixture) + }) +} + +// TestHeartbeaterRearmKeepsAConcurrentWrite is the LatentLabsSpace/NEXUS#92 +// regression for the re-arm's scoped commit: another session commits a change +// to a different issues row while the re-arm's transaction is open, and the +// Dolt commit the re-arm makes must stage issues as committed, with that +// change in it, not as the re-arm's start snapshot. DOLT_ADD stages a whole +// table from the session's transaction root, so staging inside the open +// transaction did the latter: HEAD then disagreed with the rows every reader +// sees, for a write still only in the working set (batch mode, or the window +// before the writer's own Dolt commit). ScopedCommitter commits the SQL +// transaction first. +func TestHeartbeaterRearmKeepsAConcurrentWrite(t *testing.T) { + ctx := t.Context() + const prefix = "hbw" + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + kit := newUOWRoleFixtureKit(provider, prefix) + lifecycle, err := provider.(IssueLifecycleSource).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + const holder = "hb-holder" + for _, tc := range []struct { + name string + write func(ctx context.Context, id, title string) error + }{ + {"a write with its own dolt commit", func(ctx context.Context, id, title string) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + _, err := uw.RawSQLUseCase().Exec(ctx, "UPDATE issues SET title = ? WHERE id = ?", title, id) + return "bd: retitle " + id, err + }) + }}, + {"a write only in the working set", func(ctx context.Context, id, title string) error { + _, err := RunTxEphemeral(ctx, provider, func(ctx context.Context, uw UnitOfWork) (struct{}, error) { + _, err := uw.RawSQLUseCase().Exec(ctx, "UPDATE issues SET title = ? WHERE id = ?", title, id) + return struct{}{}, err + }) + return err + }}, + } { + t.Run(tc.name, func(t *testing.T) { + suffix := strings.ReplaceAll(tc.name, " ", "-") + doled, bystander := prefix+"-doled-"+suffix, prefix+"-bystander-"+suffix + for _, id := range []string{doled, bystander} { + issue := &types.Issue{ID: id, Title: "seed " + id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := kit.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if _, err := lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: holder, IssueID: doled, + Patch: publicops.IssuePatch{ + Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusInProgress}, + Assignee: publicops.Field[string]{Set: true, Value: holder}, + }, + }); err != nil { + t.Fatalf("hand-dole %s: %v", doled, err) + } + + const retitled = "retitled while the re-arm was open" + wrote := false + heartbeater, err := NewHeartbeater(&beforeScopedCommitProvider{ + UnitOfWorkProvider: provider, + before: func(ctx context.Context) error { + if wrote { + return nil + } + wrote = true + return tc.write(ctx, bystander, retitled) + }, + }) + if err != nil { + t.Fatalf("NewHeartbeater: %v", err) + } + result, err := heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: holder, IssueID: doled}) + if err != nil { + t.Fatalf("Heartbeat: %v", err) + } + if !result.Rearmed || !wrote { + t.Fatalf("result = %+v, concurrent write ran = %v: the case needs a re-arm with a write inside its window", result, wrote) + } + + for _, at := range []string{"the working set", "HEAD"} { + query := "SELECT title FROM issues WHERE id = ?" + if at == "HEAD" { + query = "SELECT title FROM issues AS OF 'HEAD' WHERE id = ?" + } + var title string + if err := kit.QueryScalar(ctx, query, []any{bystander}, &title); err != nil { + t.Fatalf("read %s title at %s: %v", bystander, at, err) + } + if title != retitled { + t.Errorf("%s title at %s = %q, want %q: the re-arm's Dolt commit staged issues from its start snapshot", bystander, at, title, retitled) + } + } + // The re-arm itself is committed: its reminted row_lock is at HEAD + // (events is dolt_ignored, so the issues row is what its commit + // carries). + var working, head int64 + if err := kit.QueryScalar(ctx, "SELECT row_lock FROM issues WHERE id = ?", []any{doled}, &working); err != nil { + t.Fatalf("read %s row_lock: %v", doled, err) + } + if err := kit.QueryScalar(ctx, "SELECT row_lock FROM issues AS OF 'HEAD' WHERE id = ?", []any{doled}, &head); err != nil { + t.Fatalf("read %s row_lock at HEAD: %v", doled, err) + } + var commits int + if err := kit.QueryScalar(ctx, "SELECT COUNT(*) FROM dolt_log WHERE message = ?", []any{"bd: re-arm lease " + doled}, &commits); err != nil { + t.Fatalf("count re-arm commits: %v", err) + } + if head != working || commits != 1 { + t.Errorf("%s row_lock at HEAD = %d (working set %d), re-arm commits = %d, want the re-arm in one Dolt commit", doled, head, working, commits) + } + }) + } +} + +// beforeScopedCommitProvider runs before ahead of each scoped commit of the +// units of work it opens: a way into the window between a transaction's +// writes and its commit. +type beforeScopedCommitProvider struct { + UnitOfWorkProvider + before func(context.Context) error +} + +func (p *beforeScopedCommitProvider) NewUOW(ctx context.Context) (UnitOfWork, error) { + uw, err := p.UnitOfWorkProvider.NewUOW(ctx) + if err != nil { + return nil, err + } + return &beforeScopedCommitUOW{UnitOfWork: uw, before: p.before}, nil +} + +type beforeScopedCommitUOW struct { + UnitOfWork + before func(context.Context) error +} + +func (u *beforeScopedCommitUOW) Unwrap() UnitOfWork { return u.UnitOfWork } + +func (u *beforeScopedCommitUOW) CommitScoped(ctx context.Context, tables []string, message string) error { + if err := u.before(ctx); err != nil { + return err + } + return commitScoped(ctx, u.UnitOfWork, tables, message) +} + +// TestHeartbeaterRefusesBeforeOpeningAUOW: a malformed heartbeat is a +// deterministic validation failure and must not cost a database connection to +// discover, as on the dolt and embedded-dolt legs. +func TestHeartbeaterRefusesBeforeOpeningAUOW(t *testing.T) { + for _, tc := range []struct { + name string + request publicops.HeartbeatRequest + }{ + {"no actor", publicops.HeartbeatRequest{IssueID: "x"}}, + {"no id", publicops.HeartbeatRequest{Actor: "worker"}}, + } { + t.Run(tc.name, func(t *testing.T) { + provider := &mockUnitOfWorkProvider{newUOWErr: errors.New("unexpected unit-of-work open")} + heartbeater, err := NewHeartbeater(provider) + if err != nil { + t.Fatalf("NewHeartbeater: %v", err) + } + if _, err := heartbeater.Heartbeat(context.Background(), tc.request); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("err = %v, want ErrValidation", err) + } + if provider.newUOWCalls != 0 { + t.Errorf("an invalid request opened %d units of work", provider.newUOWCalls) + } + }) + } +} + +// newUOWHeartbeaterFixture takes every role through the provider's own +// capability accessors, and the hooked heartbeater off the notifying provider +// the proxied CLI route wraps around its provider. +func newUOWHeartbeaterFixture(t *testing.T, ctx context.Context, prefix string) conformance.HeartbeaterFixture { + t.Helper() + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + source, ok := provider.(HeartbeaterSource) + if !ok { + t.Fatalf("provider %T does not offer the Heartbeater accessor", provider) + } + heartbeater, err := source.Heartbeater() + if err != nil { + t.Fatalf("Heartbeater(): %v", err) + } + claimer, err := provider.(IssueClaimerSource).IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := provider.(IssueLifecycleSource).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + releaser, err := provider.(ReleaserSource).Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + reclaimer, err := provider.(LeaseReclaimerSource).LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + configurer, ok := provider.(storage.EventsJournalConfigurer) + if !ok { + t.Fatalf("provider %T does not implement storage.EventsJournalConfigurer", provider) + } + versionConfigurer, ok := provider.(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("provider %T does not implement storage.VersionedHistoryConfigurer", provider) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := heartbeater + if runner != nil { + notifying, ok := NewNotifyingProvider(provider, Sinks{Hook: runner}).(HeartbeaterSource) + if !ok { + t.Fatal("the notifying provider does not offer the Heartbeater accessor") + } + if hooked, err = notifying.Heartbeater(); err != nil { + t.Fatalf("hooked Heartbeater(): %v", err) + } + } + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.HeartbeaterFixture{ + IssuePrefix: kit.IssuePrefix, + Heartbeater: heartbeater, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + Claimer: claimer, + Lifecycle: lifecycle, + Releaser: releaser, + LeaseReclaimer: reclaimer, + QueryScalar: kit.QueryScalar, + CountHistory: kit.CountHistory, + ExecUncommitted: func(ctx context.Context, statements []conformance.SQLStatement) error { + _, err := RunTxEphemeral(ctx, provider, func(ctx context.Context, uw UnitOfWork) (struct{}, error) { + for _, stmt := range statements { + if _, err := uw.RawSQLUseCase().Exec(ctx, stmt.Query, stmt.Args...); err != nil { + return struct{}{}, fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return struct{}{}, nil + }) + return err + }, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + for _, stmt := range statements { + if _, err := uw.RawSQLUseCase().Exec(ctx, stmt.Query, stmt.Args...); err != nil { + return "", fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return "seed heartbeater rows", nil + }) + }, + SetJournalEnabled: configurer.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: versionConfigurer.SetVersionedHistoryEnabled, + HookedHeartbeater: hooked, + FiredUpdates: fired, + } +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index 1d9ee1e0f..64b934937 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -308,6 +308,14 @@ func (p *notifyingProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { return NewLeaseReclaimer(p) } +// Heartbeater builds the lease refresh over THIS provider. recordingIssueUC +// declares RenewLease: a refresh records nothing (it writes only the +// ephemeral lease row), and a re-arm records an update, the hook +// storage.HookFiringStore.Heartbeater fires for it on the DoltStorage chain. +func (p *notifyingProvider) Heartbeater() (publicops.Heartbeater, error) { + return NewHeartbeater(p) +} + func (p *notifyingProvider) Memories() (memoryops.Memories, error) { return NewMemories(p) } // EventsJournalCursor builds on THIS provider, like every role above it, so a @@ -424,6 +432,7 @@ var ( _ VersionReconcilerSource = (*notifyingProvider)(nil) _ MemoriesSource = (*notifyingProvider)(nil) _ LeaseReclaimerSource = (*notifyingProvider)(nil) + _ HeartbeaterSource = (*notifyingProvider)(nil) _ EventsJournalCursorSource = (*notifyingProvider)(nil) ) @@ -488,6 +497,22 @@ func (u *notifyingUOW) Commit(ctx context.Context, message string) error { return nil } +// CommitScoped is Commit for a scoped version commit (see ScopedCommitter): +// commit through the chain beneath, then fire the buffered hooks. +func (u *notifyingUOW) CommitScoped(ctx context.Context, tables []string, message string) error { + if err := commitScoped(ctx, u.UnitOfWork, tables, message); err != nil { + return err + } + entries := u.rec.drain() + if u.sinks.Hook == nil { + return nil + } + for _, entry := range entries { + u.sinks.Hook.Run(entry.event, entry.issue) + } + return nil +} + // Close rolls the unit of work back unless it committed, so whatever the // attempt buffered is dropped rather than fired. func (u *notifyingUOW) Close(ctx context.Context) { @@ -1009,6 +1034,20 @@ func (u *recordingIssueUC) ReleaseIssue(ctx context.Context, req publicops.Relea return result, wrote, nil } +// RenewLease records an update for a heartbeat that re-armed a lease (a +// mutation: row_lock, updated_at, event, journal, version) and nothing for a +// refresh, which writes only the ephemeral lease row. It is DECLARED for +// ReleaseIssue's reason: an inherited accessor would compile and record +// nothing, silently dropping the re-arm's hook. +func (u *recordingIssueUC) RenewLease(ctx context.Context, request publicops.HeartbeatRequest) (publicops.HeartbeatResult, error) { + result, err := u.IssueUseCase.RenewLease(ctx, request) + if err != nil || !result.Rearmed { + return result, err + } + u.rec.record(opUpdate, u.snap.issue(ctx, result.IssueID)) + return result, nil +} + // Reclaim records an update for every lease the sweep reverted. // // It is DECLARED for the same reason ReleaseIssue above is: an inherited diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index f1a9a31fd..72faa8e03 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -130,7 +130,9 @@ func TestRecordingIssueUseCaseCoversItsSurface(t *testing.T) { "has no name for a deletion and the row is gone (hook_deleter.go, hookTrackingTransaction.DeleteIssue)" leases := "lease and claim bookkeeping the DoltStorage plumbing fires no hook for: " + "HookFiringStore overrides neither UnclaimIssue, UnclaimIssueIfAssignee, HeartbeatIssue nor " + - "ReclaimExpiredLeases, so `bd unclaim` and `bd heartbeat` run no script on either plumbing" + "ReclaimExpiredLeases, so `bd unclaim` runs no script on either plumbing, and neither does a " + + "heartbeat through these doors (`bd heartbeat` takes the Heartbeater role, whose re-arm records " + + "an update through RenewLease)" assertPartition(t, "IssueUseCase", interfaceMethods(reflect.TypeOf((*domain.IssueUseCase)(nil)).Elem()), reflect.TypeOf((*recordingIssueUC)(nil)), map[string]string{ diff --git a/internal/storage/uow/tx.go b/internal/storage/uow/tx.go index 35cb0a94c..ba5b21b36 100644 --- a/internal/storage/uow/tx.go +++ b/internal/storage/uow/tx.go @@ -181,19 +181,20 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap } // RunTxEphemeral is RunTxResult for work whose writes touch ONLY ephemeral -// (dolt_ignored) state — today the leases table, whose writes must mint no -// Dolt commit and no history (bd-lrgn1; ported to proxied mode by bd-aq0ql). -// On success the attempt is committed with the SQL-only form (Tx.Commit with -// an empty message ⇒ plain COMMIT, no DOLT_COMMIT), which persists the -// working set but records nothing in dolt_log — the proxied analog of the -// classic DoltStore.withRetryTx commit discipline that HeartbeatIssue relies -// on. A serialization loser (heartbeat and reclaim/close contend on the same -// lease row by design) is replayed against a fresh unit of work, exactly like -// RunTxResult. +// (dolt_ignored) state, which must mint no Dolt commit and no history +// (bd-aq0ql) — today local metadata and the events-journal prune. On success +// the attempt is committed with the SQL-only form (Tx.Commit with an empty +// message ⇒ plain COMMIT, no DOLT_COMMIT), which persists the working set but +// records nothing in dolt_log. A serialization loser is replayed against a +// fresh unit of work, exactly like RunTxResult. // // Do NOT use this for work that writes versioned tables: those writes would // persist while silently bypassing Dolt history. Every versioned write goes -// through RunTx/RunTxResult with a real commit message. +// through RunTx/RunTxResult with a real commit message. A heartbeat is NOT +// ephemeral work: a lease refresh writes only the dolt_ignored leases table +// (bd-lrgn1), but a re-arm writes versioned rows, so the Heartbeater commits +// through RunTxScoped, whose attempt that names no tables takes this same +// plain COMMIT. func RunTxEphemeral[T any](ctx context.Context, p UnitOfWorkProvider, work TxReadFunc[T]) (T, error) { var result T bo := backoff.NewExponentialBackOff() @@ -232,6 +233,99 @@ func RunTxEphemeral[T any](ctx context.Context, p UnitOfWorkProvider, work TxRea return result, err } +// ScopedCommitter is the optional commit a unit of work offers for a write +// whose versioned tables are KNOWN: it commits the SQL transaction, then +// stages exactly tables (plus the versioned-history tables when the +// transaction could have minted) and makes a Dolt commit named message from +// that staged set alone, leaving another writer's uncommitted change to any +// other table in the working set where it was. A staged table is committed +// whole, so another writer's uncommitted change to one of those tables rides +// in this commit, as it does in the DoltStore funnel. With nothing staged, or +// under a deferred version commit, there is no Dolt commit. It is the uow +// twin of the DoltStore funnel's post-transaction DOLT_ADD-per-table commit, +// in the same order — the Dolt commit after the SQL COMMIT, never inside the +// transaction, where it would be built from the transaction's start snapshot +// (LatentLabsSpace/NEXUS#92) — and what a mutation that mostly touches +// dolt_ignored state (a heartbeat re-arm) commits through instead of +// DOLT_COMMIT('-Am'). An error means the SQL transaction did not commit; a +// Dolt commit that fails after it is logged, never returned. +type ScopedCommitter interface { + CommitScoped(ctx context.Context, tables []string, message string) error +} + +// commitScoped commits uw through the first ScopedCommitter on its decorator +// chain. A backend whose unit of work has none (a non-Dolt engine) gets the +// plain SQL COMMIT: there is no version control to scope. +func commitScoped(ctx context.Context, uw UnitOfWork, tables []string, message string) error { + for u := uw; u != nil; { + if scoped, ok := u.(ScopedCommitter); ok { + return scoped.CommitScoped(ctx, tables, message) + } + unwrapper, ok := u.(unitOfWorkUnwrapper) + if !ok { + break + } + u = unwrapper.Unwrap() + } + return uw.Commit(ctx, "") +} + +// TxScopedFunc is the work RunTxScoped runs: its result, and the versioned +// tables the attempt changed with the commit message naming them. No tables +// means the attempt wrote only dolt_ignored state (or nothing). +type TxScopedFunc[T any] func(ctx context.Context, uw UnitOfWork) (result T, tables []string, commitMsg string, err error) + +// RunTxScoped is RunTxEphemeral for work that is USUALLY ephemeral but +// sometimes writes versioned rows it can name: an attempt that reports no +// tables commits with the plain SQL COMMIT (no working-set scan, no Dolt +// commit, nothing swept); one that reports tables commits through +// ScopedCommitter, staging exactly those. Never DOLT_COMMIT('-Am'): that +// would sweep other writers' uncommitted (batch-mode) changes to other tables +// into this commit (a staged table is committed whole). Serialization losers +// are replayed like RunTxEphemeral's. +func RunTxScoped[T any](ctx context.Context, p UnitOfWorkProvider, work TxScopedFunc[T]) (T, error) { + var result T + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = txRetryInitialInterval + bo.MaxElapsedTime = DefaultTxRetryMaxElapsed + + err := backoff.Retry(func() error { + uw, err := p.NewUOW(ctx) + if err != nil { + if isSerializationError(err) { + return err + } + return backoff.Permanent(err) + } + defer closeAttempt(ctx, uw) + + r, tables, msg, err := work(ctx, uw) + if err != nil { + if isSerializationError(err) { + return err + } + return backoff.Permanent(err) + } + + if len(tables) == 0 || msg == "" { + err = uw.Commit(ctx, "") + } else { + err = commitScoped(ctx, uw, tables, msg) + } + if err != nil { + if isSerializationError(err) { + return err + } + return backoff.Permanent(err) + } + + result = r + return nil + }, backoff.WithContext(bo, ctx)) + + return result, err +} + func RunTxRead[T any](ctx context.Context, p UnitOfWorkProvider, work TxReadFunc[T]) (T, error) { var zero T uw, err := p.NewUOW(ctx) diff --git a/internal/storage/uow/uow.go b/internal/storage/uow/uow.go index a141fa5bb..ab69586c5 100644 --- a/internal/storage/uow/uow.go +++ b/internal/storage/uow/uow.go @@ -55,6 +55,20 @@ func (u *baseUOW) Commit(ctx context.Context, message string) error { return u.tx.Commit(ctx, message) } +// CommitScoped commits through the transaction's scoped commit when it has +// one (the Dolt server transaction), else with the plain SQL COMMIT. +func (u *baseUOW) CommitScoped(ctx context.Context, tables []string, message string) error { + if scoped, ok := u.tx.(scopedCommitTx); ok { + return scoped.CommitScoped(ctx, tables, message) + } + return u.tx.Commit(ctx, "") +} + +// scopedCommitTx is the optional Tx half of ScopedCommitter. +type scopedCommitTx interface { + CommitScoped(ctx context.Context, tables []string, message string) error +} + func (u *baseUOW) Close(ctx context.Context) { u.tx.RollbackUnlessCommitted(ctx) } diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index e1446ce6c..654290d48 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "dependency_editor.go", "edgereader.go", "graph_counter.go", + "heartbeater.go", "init_verifier.go", "issue_claimer.go", "issue_operations.go", diff --git a/internal/telemetry/heartbeater.go b/internal/telemetry/heartbeater.go new file mode 100644 index 000000000..e0be3a106 --- /dev/null +++ b/internal/telemetry/heartbeater.go @@ -0,0 +1,36 @@ +package telemetry + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// Heartbeater returns the inner store's lease-refresh surface wrapped in this +// layer's instrumentation. It recurses instead of delegating: a blind +// delegation would return the inner surface unspanned and untimed. +func (s *InstrumentedStorage) Heartbeater() (issueops.Heartbeater, error) { + inner, err := s.Unwrap().Heartbeater() + if err != nil { + return nil, err + } + return s.WrapHeartbeater(inner), nil +} + +// WrapHeartbeater instruments the lease refresh with this storage layer's +// existing telemetry meter and tracer. +func (s *InstrumentedStorage) WrapHeartbeater(inner issueops.Heartbeater) issueops.Heartbeater { + return &instrumentedHeartbeater{storage: s, inner: inner} +} + +type instrumentedHeartbeater struct { + storage *InstrumentedStorage + inner issueops.Heartbeater +} + +func (c *instrumentedHeartbeater) Heartbeat(ctx context.Context, request issueops.HeartbeatRequest) (result issueops.HeartbeatResult, err error) { + ctx, span, started := c.storage.op(ctx, "Heartbeater.Heartbeat") + result, err = c.inner.Heartbeat(ctx, request) + c.storage.done(ctx, span, started, err) + return result, err +} diff --git a/internal/telemetry/role_accessor_decorator_test.go b/internal/telemetry/role_accessor_decorator_test.go index 24a2d953b..24b5d2f3f 100644 --- a/internal/telemetry/role_accessor_decorator_test.go +++ b/internal/telemetry/role_accessor_decorator_test.go @@ -94,7 +94,7 @@ func TestInstrumentedStorageDeclaresEveryRoleAccessor(t *testing.T) { } // roleAccessorStore is a DoltStorage whose only real methods are the -// twenty-nine role accessors, each answering with a distinguishable sentinel +// thirty role accessors, each answering with a distinguishable sentinel // so a test can tell an instrumented surface from a passed-through one. // // TWO sentinels rather than one: memoryops.Memories.List and issueops.Reader.List @@ -178,8 +178,11 @@ func (s *roleAccessorStore) Releaser() (issueops.Releaser, error) { func (s *roleAccessorStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return s.surface, s.err } +func (s *roleAccessorStore) Heartbeater() (issueops.Heartbeater, error) { + return s.surface, s.err +} -// roleAccessorSentinel implements twenty-eight of the twenty-nine roles at +// roleAccessorSentinel implements twenty-nine of the thirty roles at // once — every one but memoryops.Memories, whose List collides with // issueops.Reader.List and needs the second sentinel below. // Nothing calls its methods; identity is the whole point. @@ -312,6 +315,9 @@ func (*roleAccessorSentinel) Release(context.Context, issueops.ReleaseRequest) ( func (*roleAccessorSentinel) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { return issueops.ReclaimResult{}, nil } +func (*roleAccessorSentinel) Heartbeat(context.Context, issueops.HeartbeatRequest) (issueops.HeartbeatResult, error) { + return issueops.HeartbeatResult{}, nil +} // memoryRoleSentinel is the memory role's sentinel — see // memoryRoleSentinel is the remaining role's sentinel — see @@ -389,6 +395,7 @@ func TestInstrumentedStorageInstrumentsEveryRoleAccessor(t *testing.T) { {"BatchApplier", func() (any, error) { return wrapped.BatchApplier() }, sentinel}, {"Releaser", func() (any, error) { return wrapped.Releaser() }, sentinel}, {"LeaseReclaimer", func() (any, error) { return wrapped.LeaseReclaimer() }, sentinel}, + {"Heartbeater", func() (any, error) { return wrapped.Heartbeater() }, sentinel}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() @@ -476,6 +483,7 @@ func TestInstrumentedStorageRoleAccessorsPropagateInnerErrors(t *testing.T) { {"BatchApplier", func() (any, error) { return wrapped.BatchApplier() }}, {"Releaser", func() (any, error) { return wrapped.Releaser() }}, {"LeaseReclaimer", func() (any, error) { return wrapped.LeaseReclaimer() }}, + {"Heartbeater", func() (any, error) { return wrapped.Heartbeater() }}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() diff --git a/internal/types/types.go b/internal/types/types.go index b5461c910..45efe6938 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1849,6 +1849,11 @@ const ( // EventLeaseReclaimed records that a stale lease was reverted to ready by // bd reclaim (dead-worker recovery). old_value is the previous owner. EventLeaseReclaimed EventType = "lease_reclaimed" + // EventLeaseRearmed records that a heartbeat re-armed a claim that held no + // lease row (a claim hand-doled through a generic update): the lease was + // granted and row_lock reminted. new_value is the holder. A plain lease + // refresh records no event. + EventLeaseRearmed EventType = "lease_rearmed" ) // ProvenanceEvent is one entry in the append-only provenance log: a typed diff --git a/issue_roles_external_test.go b/issue_roles_external_test.go index c9fde9947..f6e067fda 100644 --- a/issue_roles_external_test.go +++ b/issue_roles_external_test.go @@ -846,3 +846,35 @@ func TestDeleterExposesTypedUnsupportedError(t *testing.T) { t.Fatalf("Deleter() error = %v, want *beads.ErrUnsupported", err) } } + +// TestHeartbeaterLayersHooksOutsideTelemetry is the same pin for the lease +// refresh. It gives the release's answer: a refresh fires nothing, but a +// re-arm is recorded as a mutation and fires on_update +// (internal/storage/hook_heartbeater.go), so this decorator wraps rather than +// recursing the way Sweeper and Deleter do. +func TestHeartbeaterLayersHooksOutsideTelemetry(t *testing.T) { + t.Setenv("BD_OTEL_STDOUT", "true") + instrumented, ok := telemetry.WrapStorage(&dolt.DoltStore{}).(*telemetry.InstrumentedStorage) + if !ok { + t.Fatal("WrapStorage() did not create InstrumentedStorage") + } + + heartbeater, err := storage.NewHookFiringStore(instrumented, nil).Heartbeater() + if err != nil { + t.Fatalf("Heartbeater() error = %v", err) + } + if got := reflect.TypeOf(heartbeater).String(); got != "*storage.hookHeartbeater" { + t.Fatalf("outer layer = %s, want the hook wrapper", got) + } +} + +func TestHeartbeaterExposesTypedUnsupportedError(t *testing.T) { + heartbeater, err := (*dolt.DoltStore)(nil).Heartbeater() + if heartbeater != nil { + t.Fatalf("Heartbeater() heartbeater = %T, want nil", heartbeater) + } + var unsupported *beads.ErrUnsupported + if !errors.As(err, &unsupported) { + t.Fatalf("Heartbeater() error = %v, want *beads.ErrUnsupported", err) + } +} diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 62fec725e..1ebb74326 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "export.go", "gates.go", "graphcounter.go", + "heartbeater.go", "importer.go", "initverifier.go", "issueops.go", @@ -61,6 +62,7 @@ go_test( "current_test.go", "export_test.go", "gates_test.go", + "heartbeater_test.go", "issueops_external_test.go", "molecule_internal_test.go", "molecule_test.go", diff --git a/issueops/heartbeater.go b/issueops/heartbeater.go new file mode 100644 index 000000000..7129c1f89 --- /dev/null +++ b/issueops/heartbeater.go @@ -0,0 +1,182 @@ +package issueops + +import ( + "context" + "fmt" + "time" +) + +// ErrLeaseNotHeld is the refusal a heartbeat answers when the caller holds no +// live claim on the issue: it is open, closed, released, reclaimed, moved off +// in_progress, or a wisp (wisps are never leased). +// +// IT ALSO MATCHES ErrNotClaimable, which is what the raw heartbeat has always +// answered for the same states, so a caller written against that sentinel +// keeps working when it moves onto this role. +var ErrLeaseNotHeld = fmt.Errorf("lease not held: %w", ErrNotClaimable) + +// LeaseConflictError reports the state that refused a heartbeat, read inside +// the same transaction that found no lease row for the caller. It wraps the +// refusal rather than replacing it, the way ClaimConflictError does, so the +// sentinel still matches and a front door can classify the conflict from +// typed fields instead of parsing prose. +type LeaseConflictError struct { + // IssueID names the issue the heartbeat was refused on. + IssueID string + // Holder is the foreign assignee that holds the claim. It is empty when + // the refusal is about the status (no live claim) rather than a foreign + // holder. + Holder string + // Status is the status the refusing transaction observed. + Status Status + // Ephemeral is true when the id names a wisp, which is never leased. + // Status is then the wisp's own status, which may well be in_progress. + Ephemeral bool + // Err is the wrapped refusal. It matches ErrAlreadyClaimed (a foreign + // holder) or ErrLeaseNotHeld (no live claim by the caller). + Err error +} + +func (e *LeaseConflictError) Error() string { return e.Err.Error() } + +// Unwrap makes LeaseConflictError match the refusal it carries. +func (e *LeaseConflictError) Unwrap() error { return e.Err } + +// The HeartbeatRequest fields a HeartbeatFieldError can name. +const ( + HeartbeatFieldActor = "actor" + HeartbeatFieldIssueID = "issue_id" +) + +// HeartbeatFieldError reports a HeartbeatRequest refusal and the one field it +// is about, so a front door can name that field without parsing the message. +// It wraps ErrValidation. +type HeartbeatFieldError struct { + // Field is one of the HeartbeatField* constants. + Field string + // Detail is the sentence the error reads as. + Detail string +} + +func (e *HeartbeatFieldError) Error() string { return e.Detail } + +// Unwrap makes HeartbeatFieldError match ErrValidation. +func (e *HeartbeatFieldError) Unwrap() error { return ErrValidation } + +// HeartbeatRequest refreshes the lease on one issue the actor holds — the +// shape behind `bd heartbeat`. +// +// Implementations never mutate caller-owned request values. +type HeartbeatRequest struct { + // Actor is the lease holder refreshing its lease, and the fence's + // subject: only the current holder may heartbeat. It is REQUIRED — a + // blank Actor is ErrValidation — and so is one longer than + // types.MaxFieldLen characters, which no lease row can hold. + // + // THE ROLE DOES NOT TRIM OR CASE-FOLD IT, Releaser.Actor's rule: the + // value is compared with the lease row's holder as sent. A front door may + // normalize before it calls the role: bd serve trims the body's actor by + // the claim's rule (so a served claim and a served heartbeat agree on the + // spelling), and the CLI passes its resolved actor through. A holder + // spelled differently across layers (separator-insensitive, see + // ReleaseRequest.ExpectedAssignee) still heartbeats: it misses the fast + // path, and the slow path refreshes the lease row in place under this + // spelling. + Actor string + // IssueID is the exact canonical id, on either plane, and must not be + // empty. There is no fuzzy or prefix resolution: that is a front-door + // convenience, for the reason GetRequest.ID gives. + IssueID string +} + +// HeartbeatResult reports one refreshed lease. +type HeartbeatResult struct { + // IssueID is the issue whose lease was refreshed. + IssueID string + // Holder is the lease holder after the refresh: the request's Actor, in + // the request's spelling. + Holder string + // HeartbeatAt is the server clock, in UTC and rounded to whole seconds + // as the lease row's column rounds it: the stored value. + HeartbeatAt time.Time + // LeaseExpiresAt is HeartbeatAt plus the store's lease TTL: the deadline + // the next heartbeat must beat before a reclaim may revert the claim. + LeaseExpiresAt time.Time + // Rearmed is true only when the heartbeat CREATED the caller's lease row: + // the caller held a live claim but no lease row the caller holds existed + // (a claim hand-doled through a generic update), so the heartbeat granted + // a fresh one instead of refreshing it. A re-arm is recorded as a + // mutation and remints the issue's revision token (see Heartbeater); + // re-read before a guarded write. A refresh of the caller's own row + // is never a re-arm, including one that changed no value (two heartbeats + // in one second) and one under a differently spelled holder. + // + // false does not prove that nothing reminted the token: a transport that + // resends a heartbeat whose answer was lost (the HTTP client does) reports + // the resend's answer, a refresh of the lease row a committed re-arm + // granted. A guarded write composed on the old revision then fails its + // revision check once and re-reads. + Rearmed bool +} + +// Heartbeater refreshes a held lease — the capability behind `bd heartbeat` — +// and, like every other capability here, a role with its own accessor. +// +// IT IS A DIFFERENT QUESTION FROM Claimer. A claim takes ownership and is +// idempotent for the holder WITHOUT extending the lease (a re-claim returns +// before the lease is touched); a heartbeat proves the holder is alive and +// does nothing else. Folding one into the other would make every keepalive a +// claim attempt, with a claim's refusals and a claim's history. +// +// A REFRESH WRITES ONLY THE LEASE ROW. A heartbeat touches the ephemeral +// leases table and nothing else: it does not remint row_lock (so the holder's +// own revision token survives it and its guarded writes keep working), it +// records no version (current_revision and issue_versions stay put), it does +// not stamp updated_at, it records no event and no events-journal entry, it +// makes no version-control commit, and it fires no hook. It is the hottest +// write in a fleet, and every one of those would make it expensive or noisy. +// +// A RE-ARM IS RECORDED AS A MUTATION; A REFRESH IS NOT. A re-arm (Rearmed: +// the caller held a live claim with no lease row, a claim hand-doled through +// a generic update) is recorded the way every bead mutator records a write: +// it remints row_lock, stamps updated_at, records a lease_rearmed event, an +// events-journal entry and a version, makes a version-control commit, and +// fires on_update. The remint is what makes the grant collide with a release +// or close racing it, instead of committing beside it and leaving a lease on +// an issue that is no longer claimed. Re-arms are rare (a hand-doled claim's +// first heartbeat), so the cost lands once per such claim. +// +// THE LIVE-CLAIM FENCE IS AN INVARIANT AND A PREDICATE. Every path that ends +// or transfers a claim deletes the lease row, and a re-arm racing such a path +// collides with it on row_lock, so a lease row exists only while its claim is +// live. The refresh also matches the lease row by holder AND requires, in the +// same transaction, that the issue is in_progress with that holder as assignee, +// so a lease row orphaned any other way fails the refresh, and the refusal +// deletes it. A refresh that lands is a refresh of a live claim. +// +// EVERY LEG RUNS ONE BODY: dolt, embedded dolt and the unit-of-work provider +// wrap internal/storage/issueops.ExecuteHeartbeatInTx. +// +// Deterministic request-validation failures match ErrValidation. Result +// values are unspecified when error is non-nil. Every refusal below leaves +// persistent state unchanged, with one exception: a *LeaseConflictError +// refusal deletes the caller's own lease row if one survives on an issue the +// caller no longer holds a live claim on (an orphan; never a live holder's +// lease). +type Heartbeater interface { + // Heartbeat refreshes the lease on request.IssueID held by request.Actor. + // + // REFUSALS: + // + // - a blank Actor, one longer than types.MaxFieldLen characters, or an + // empty IssueID: a *HeartbeatFieldError (ErrValidation), before + // anything is read; + // - an IssueID that names no row on either plane: ErrNotFound; + // - an issue another actor holds: a *LeaseConflictError naming the + // Holder, wrapping ErrAlreadyClaimed; + // - a wisp, or an issue the actor holds no live claim on (open, closed, + // released, reclaimed, or any status but in_progress): a + // *LeaseConflictError wrapping ErrLeaseNotHeld, which also matches + // ErrNotClaimable; Ephemeral is set for a wisp. + Heartbeat(ctx context.Context, request HeartbeatRequest) (HeartbeatResult, error) +} diff --git a/issueops/heartbeater_test.go b/issueops/heartbeater_test.go new file mode 100644 index 000000000..c27e86d14 --- /dev/null +++ b/issueops/heartbeater_test.go @@ -0,0 +1,30 @@ +package issueops_test + +import ( + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestErrLeaseNotHeldStillMatchesErrNotClaimable pins the compatibility +// promise ErrLeaseNotHeld's doc makes: a caller written against the raw +// heartbeat's ErrNotClaimable keeps classifying the role's refusal. +func TestErrLeaseNotHeldStillMatchesErrNotClaimable(t *testing.T) { + err := &issueops.LeaseConflictError{IssueID: "x", Err: fmt.Errorf("%w: x status closed", issueops.ErrLeaseNotHeld)} + if !errors.Is(err, issueops.ErrLeaseNotHeld) || !errors.Is(err, issueops.ErrNotClaimable) { + t.Fatalf("LeaseConflictError(ErrLeaseNotHeld) = %v: want it to match ErrLeaseNotHeld and ErrNotClaimable", err) + } + if errors.Is(err, issueops.ErrAlreadyClaimed) { + t.Fatal("ErrLeaseNotHeld matches ErrAlreadyClaimed; the two refusals must stay distinguishable") + } +} + +// TestHeartbeatFieldErrorMatchesErrValidation pins the field error's sentinel. +func TestHeartbeatFieldErrorMatchesErrValidation(t *testing.T) { + var err error = &issueops.HeartbeatFieldError{Field: issueops.HeartbeatFieldActor, Detail: "validation failed: heartbeat actor is required"} + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("HeartbeatFieldError = %v: want it to match ErrValidation", err) + } +} From a3e1b24e2b8780a5231cc1b92b5960f929c40d8f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 23:55:53 +0000 Subject: [PATCH 04/54] fix(import): tolerate only typed dependency refusals, not walk-query errors (#7630) * fix(import): tolerate only typed dependency refusals, not walk-query errors With SkipDependencyValidationErrors set (the import mode), the dependency pass recorded ANY error from checkHierarchy/checkCycle as a skipped edge and continued. That included failed walk queries (an untranslated statement, a statement timeout, a dropped connection), so an import could commit without the edge and report the SQL error only as a skip reason; on a backend whose transaction aborts on a failed statement, the import failed later at an unrelated statement with the real cause hidden. Tolerate only the typed refusals (ErrSelfDependency, ErrDependencyCycle, *DependencyHierarchyConflictError) via isDependencyValidationRefusal and return every other error, keeping the %w chain so serialization retry and error classification apply. Tests: sqlmock cases where the hierarchy and cycle walks fail under import mode, on both the per-edge statements and the batch graph, must abort the pass; typed refusals on both shapes are still skipped. Removing the predicate fails both error tests. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(import): record the dependency-check fix in the CHANGELOG; tidy two test comments Review fixups for #7630. - CHANGELOG (Unreleased, Fixed): bd import now fails when a dependency check's own query fails, instead of reporting the SQL error as a skipped dependency and, on Dolt, committing without the edge. Refused edges (a self-dependency, a cycle, a hierarchy conflict) are still skipped. docs/ is left alone because it tracks the pinned release (v1.2.2). - create_test.go: two doc comments kept the line breaks left where the internal tracker IDs were removed. They are reflowed into sentences; no test logic changes. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 + internal/storage/batch.go | 5 +- internal/storage/issueops/create.go | 23 +- internal/storage/issueops/create_test.go | 331 +++++++++++++++++++++++ 4 files changed, 367 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 604b85bd9..1808f549a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -595,6 +595,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd import` no longer reports a failed dependency check as a skipped + dependency.** Import skips an edge the dependency checks refuse (a + self-dependency, a cycle, or a blocker that is the issue's own ancestor or + descendant) and reports it on a `Skipped dependency:` line, or in + `skipped_dependencies` with `--json`. It did the same when the check's own + query failed, for example on a timeout, a dropped connection or a statement + the backend could not run: the SQL error became the skip reason, and on + Dolt, where the transaction survives a failed statement, the import + committed without the edge and exited 0. Such an error now fails the + import. Refused edges are still skipped as before. + - **`bd mol bond ` ignored prefix routing** — operand resolution now uses the same routing fallback as `bd show`/`bd update`/`bd close`, so a bead that lives in a routed store (a prefix-routed rig via `routes.jsonl`, or the diff --git a/internal/storage/batch.go b/internal/storage/batch.go index 9093c14b5..30044d8ed 100644 --- a/internal/storage/batch.go +++ b/internal/storage/batch.go @@ -30,7 +30,10 @@ type BatchCreateOptions struct { // leave it false. RejectStaleUpserts bool // SkipDependencyValidationErrors skips dependency validation failures that - // legacy imports tolerated, such as cycles or self-dependencies. + // legacy imports tolerated: typed refusals of an edge (a cycle, a + // self-dependency, or a blocker in the issue's own hierarchy). An error + // from the validation queries themselves is still returned, never recorded + // as a skipped edge. SkipDependencyValidationErrors bool // OnSkippedDependency records dependency edges skipped during batch create. OnSkippedDependency func(issueID, dependsOnID, reason string) diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index 0200c202c..b699e9d86 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -12,6 +12,7 @@ import ( "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/depid" + "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/types" ) @@ -1145,7 +1146,7 @@ func PersistDependenciesWithOptionsResult(ctx context.Context, tx DBTX, issues [ if kind != DepTargetExternal && types.ExtractPrefix(dep.IssueID) == types.ExtractPrefix(dep.DependsOnID) { if err := lookups.checkHierarchy(ctx, tx, dep); err != nil { - if opts.SkipDependencyValidationErrors { + if opts.SkipDependencyValidationErrors && isDependencyValidationRefusal(err) { recordSkippedDependency(opts, dep, err.Error()) continue } @@ -1154,7 +1155,7 @@ func PersistDependenciesWithOptionsResult(ctx context.Context, tx DBTX, issues [ } if err := lookups.checkCycle(ctx, tx, dep); err != nil { - if opts.SkipDependencyValidationErrors { + if opts.SkipDependencyValidationErrors && isDependencyValidationRefusal(err) { recordSkippedDependency(opts, dep, err.Error()) continue } @@ -1223,6 +1224,24 @@ func dependencyCreatedBy(dep *types.Dependency, actor string) string { return actor } +// isDependencyValidationRefusal reports whether err, from checkHierarchy or +// checkCycle, is a typed refusal of the edge itself (a self-dependency, a +// cycle, or a blocker that is the issue's own ancestor or descendant) — the +// only failures SkipDependencyValidationErrors tolerates. Every other error is +// a failed walk query (an untranslated statement, a timeout, a dropped +// connection): recording it as a skipped edge would commit the import without +// the edge and report the SQL error only as a skip reason. On a backend whose +// transaction aborts on a failed statement, the import would instead fail +// later at an unrelated statement, with the real cause hidden in the skip +// reason. +func isDependencyValidationRefusal(err error) bool { + if errors.Is(err, domain.ErrSelfDependency) || errors.Is(err, domain.ErrDependencyCycle) { + return true + } + var hierarchy *domain.DependencyHierarchyConflictError + return errors.As(err, &hierarchy) +} + func recordSkippedDependency(opts storage.BatchCreateOptions, dep *types.Dependency, reason string) { if dep == nil { return diff --git a/internal/storage/issueops/create_test.go b/internal/storage/issueops/create_test.go index c47a9dedc..5cd2f8b89 100644 --- a/internal/storage/issueops/create_test.go +++ b/internal/storage/issueops/create_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "database/sql/driver" "errors" + "fmt" "regexp" "strings" "testing" @@ -12,6 +13,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/depid" + "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/types" ) @@ -623,3 +625,332 @@ func TestPersistDependenciesBatchValidatesPlannedHierarchyBeforeBlocking(t *test t.Fatalf("unmet expectations: %v", err) } } + +// A failed walk query is not a validation refusal: under the import mode +// (SkipDependencyValidationErrors) the error must abort the pass rather than be +// recorded as a skipped edge. This covers both checks on the per-edge +// statements (a one-edge batch); the batch graph's twin is +// TestPersistDependenciesBatchTolerantReturnsValidationQueryErrors. +func TestPersistDependenciesTolerantReturnsValidationQueryErrors(t *testing.T) { + walkErr := errors.New("pq: aborted walk (simulated)") + cases := []struct { + name string + depType types.DependencyType + walk string + args []driver.Value + wantMsg string + }{ + {"hierarchy", types.DepBlocks, "WITH RECURSIVE ancestors", []driver.Value{"bd-child", "bd-parent"}, "failed to check blocker ancestry"}, + // parent-child skips the hierarchy check, so its first walk is the cycle check. + {"cycle", types.DepParentChild, "WITH RECURSIVE reachable", []driver.Value{"bd-parent", "bd-child"}, "failed to check for dependency cycle"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + db, mock, tx := beginMockTx(t) + defer db.Close() + issue := &types.Issue{ + ID: "bd-child", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{ + DependsOnID: "bd-parent", + Type: tc.depType, + }}, + } + mock.ExpectQuery("SELECT 1 FROM wisps WHERE id = \\? LIMIT 1"). + WithArgs("bd-parent"). + WillReturnError(sql.ErrNoRows) + mock.ExpectQuery("SELECT 1 FROM issues WHERE id = \\?"). + WithArgs("bd-parent"). + WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + mock.ExpectQuery(tc.walk).WithArgs(tc.args...).WillReturnError(walkErr) + + var skipped []string + _, err := PersistDependenciesWithOptionsResult(ctx, tx, []*types.Issue{issue}, "tester", storage.BatchCreateOptions{ + SkipDependencyValidationErrors: true, + OnSkippedDependency: func(issueID, dependsOnID, reason string) { + skipped = append(skipped, issueID+" -> "+dependsOnID+": "+reason) + }, + }) + if !errors.Is(err, walkErr) || !strings.Contains(err.Error(), tc.wantMsg) { + t.Fatalf("error = %v, want the walk error (%q)", err, tc.wantMsg) + } + if len(skipped) != 0 { + t.Fatalf("skipped = %#v, want none: a query error is not a validation refusal", skipped) + } + + mock.ExpectRollback() + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + } +} + +func TestIsDependencyValidationRefusal(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"cycle", domain.ErrDependencyCycle, true}, + {"wrapped self", fmt.Errorf("%w: x cannot depend on itself", domain.ErrSelfDependency), true}, + {"hierarchy", &domain.DependencyHierarchyConflictError{IssueID: "a", BlockerID: "b"}, true}, + {"wrapped hierarchy", fmt.Errorf("edge: %w", &domain.DependencyHierarchyConflictError{IssueID: "a", BlockerID: "b", BlockerIsAncestor: true}), true}, + {"ancestry query", fmt.Errorf("failed to check blocker ancestry: %w", errors.New("42P19")), false}, + {"cycle query", fmt.Errorf("failed to check for dependency cycle: %w", sql.ErrConnDone), false}, + {"graph read", fmt.Errorf("failed to check for dependency cycle: load dependency edges from dependencies: %w", context.DeadlineExceeded), false}, + } + for _, tc := range cases { + if got := isDependencyValidationRefusal(tc.err); got != tc.want { + t.Errorf("%s: isDependencyValidationRefusal(%v) = %v, want %v", tc.name, tc.err, got, tc.want) + } + } +} + +// TestPersistDependenciesBatchTolerantReturnsValidationQueryErrors is the +// batch-lookup twin of TestPersistDependenciesTolerantReturnsValidationQueryErrors. +// With two scheduling edges the walks run on depGraph, which reads a node's +// edges the first time a search reaches it. A stored edge leads each walk off +// the batch's endpoints, and that read fails. The failure must abort the pass: +// load has already marked the node as read, so a pass that carried on would +// answer every later walk through that node from an empty edge set. +func TestPersistDependenciesBatchTolerantReturnsValidationQueryErrors(t *testing.T) { + walkErr := errors.New("pq: aborted walk (simulated)") + cases := []struct { + name string + stored []driver.Value // issue_id, target, type: the edge the walk follows off the endpoints + wantMsg string + }{ + // bd-a's stored parent: the ancestry walk from bd-a reads bd-p's edges. + {"hierarchy", []driver.Value{"bd-a", "bd-p", string(types.DepParentChild)}, "failed to check blocker ancestry"}, + // bd-b's stored blocker: the cycle walk from bd-b reads bd-x's edges. + {"cycle", []driver.Value{"bd-b", "bd-x", string(types.DepBlocks)}, "failed to check for dependency cycle"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + db, mock, tx := beginMockTx(t) + defer db.Close() + issue := &types.Issue{ + ID: "bd-a", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: "bd-b", Type: types.DepBlocks}, + {DependsOnID: "bd-c", Type: types.DepBlocks}, + }, + } + expectDepBatchReads(mock, []driver.Value{"bd-b", "bd-c"}, []driver.Value{"bd-a", "bd-b", "bd-c"}, tc.stored) + mock.ExpectQuery(regexp.QuoteMeta("SELECT issue_id, " + DepTargetExpr + ", type FROM dependencies WHERE issue_id IN (?)")). + WithArgs(tc.stored[1]). + WillReturnError(walkErr) + + var skipped []string + _, err := PersistDependenciesWithOptionsResult(ctx, tx, []*types.Issue{issue}, "tester", storage.BatchCreateOptions{ + SkipDependencyValidationErrors: true, + OnSkippedDependency: func(issueID, dependsOnID, reason string) { + skipped = append(skipped, issueID+" -> "+dependsOnID+": "+reason) + }, + }) + if !errors.Is(err, walkErr) || !strings.Contains(err.Error(), tc.wantMsg) { + t.Fatalf("error = %v, want the walk error (%q)", err, tc.wantMsg) + } + if len(skipped) != 0 { + t.Fatalf("skipped = %#v, want none: a query error is not a validation refusal", skipped) + } + + mock.ExpectRollback() + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + } +} + +// The import mode still tolerates every typed refusal the walks raise, on both +// lookup shapes: the edge is reported as skipped and the pass goes on. This +// pins the other side of isDependencyValidationRefusal. The self-dependency +// case is TestPersistDependenciesSkipsValidationErrorsWhenConfigured. +func TestPersistDependenciesTolerantSkipsTypedRefusals(t *testing.T) { + tolerant := func(skipped *[]string) storage.BatchCreateOptions { + return storage.BatchCreateOptions{ + SkipDependencyValidationErrors: true, + OnSkippedDependency: func(issueID, dependsOnID, reason string) { + *skipped = append(*skipped, issueID+" -> "+dependsOnID+": "+reason) + }, + } + } + finish := func(t *testing.T, mock sqlmock.Sqlmock, tx *sql.Tx) { + t.Helper() + mock.ExpectRollback() + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + } + + type walk struct { + query string + args []driver.Value + count int + } + ancestors := func(node, candidate string, count int) walk { + return walk{"WITH RECURSIVE ancestors", []driver.Value{node, candidate}, count} + } + // The per-edge statements: one blocks edge bd-a -> bd-b. + for _, tc := range []struct { + name string + walks []walk + want string + }{ + {"ancestor", []walk{ancestors("bd-a", "bd-b", 1)}, "bd-a cannot be blocked by its ancestor bd-b"}, + {"descendant", []walk{ancestors("bd-a", "bd-b", 0), ancestors("bd-b", "bd-a", 1)}, "bd-a cannot be blocked by its descendant bd-b"}, + {"cycle", []walk{ + ancestors("bd-a", "bd-b", 0), ancestors("bd-b", "bd-a", 0), + {"WITH RECURSIVE reachable", []driver.Value{"bd-b", "bd-a"}, 1}, + }, domain.ErrDependencyCycle.Error()}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + db, mock, tx := beginMockTx(t) + defer db.Close() + issue := &types.Issue{ + ID: "bd-a", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{DependsOnID: "bd-b", Type: types.DepBlocks}}, + } + mock.ExpectQuery("SELECT 1 FROM wisps WHERE id = \\? LIMIT 1"). + WithArgs("bd-b"). + WillReturnError(sql.ErrNoRows) + mock.ExpectQuery("SELECT 1 FROM issues WHERE id = \\?"). + WithArgs("bd-b"). + WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + for _, w := range tc.walks { + mock.ExpectQuery(w.query).WithArgs(w.args...). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(w.count)) + } + + var skipped []string + if _, err := PersistDependenciesWithOptionsResult(ctx, tx, []*types.Issue{issue}, "tester", tolerant(&skipped)); err != nil { + t.Fatalf("error = %v, want the refusal skipped", err) + } + if len(skipped) != 1 || !strings.HasPrefix(skipped[0], "bd-a -> bd-b: "+tc.want) { + t.Fatalf("skipped = %#v, want bd-a -> bd-b: %s", skipped, tc.want) + } + finish(t, mock, tx) + }) + } + + // The batch graph: the planned ancestry of + // TestPersistDependenciesBatchValidatesPlannedHierarchyBeforeBlocking, whose + // blocks edge is skipped here instead of failing the batch. + t.Run("batch ancestor", func(t *testing.T) { + ctx := context.Background() + db, mock, tx := beginMockTx(t) + defer db.Close() + child := &types.Issue{ + ID: "bd-child", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: "bd-grand", Type: types.DepBlocks}, + {DependsOnID: "bd-parent", Type: types.DepParentChild}, + }, + } + parent := &types.Issue{ + ID: "bd-parent", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{DependsOnID: "bd-grand", Type: types.DepParentChild}}, + } + expectDepBatchReads(mock, []driver.Value{"bd-grand", "bd-parent"}, []driver.Value{"bd-child", "bd-grand", "bd-parent"}) + for _, pair := range [][2]string{{"bd-child", "bd-parent"}, {"bd-parent", "bd-grand"}} { + mock.ExpectExec("INSERT INTO dependencies"). + WithArgs(depid.New(pair[0], pair[1]), pair[0], pair[1], types.DepParentChild, "tester", sqlmock.AnyArg(), "{}", ""). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("REPLACE INTO local_metadata"). + WithArgs(dependencyCoordinationKey(pair[1], dependencyCoordinationDurableTier), sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(1, 1)) + } + + var skipped []string + if _, err := PersistDependenciesWithOptionsResult(ctx, tx, []*types.Issue{child, parent}, "tester", tolerant(&skipped)); err != nil { + t.Fatalf("error = %v, want the refusal skipped", err) + } + if len(skipped) != 1 || !strings.HasPrefix(skipped[0], "bd-child -> bd-grand: bd-child cannot be blocked by its ancestor bd-grand") { + t.Fatalf("skipped = %#v, want the planned-ancestor refusal of bd-child -> bd-grand", skipped) + } + finish(t, mock, tx) + }) + + // The batch graph: an in-batch cycle. The first edge lands, and the graph + // that now holds it refuses the second. + t.Run("batch cycle", func(t *testing.T) { + ctx := context.Background() + db, mock, tx := beginMockTx(t) + defer db.Close() + a := &types.Issue{ + ID: "bd-a", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{DependsOnID: "bd-b", Type: types.DepBlocks}}, + } + b := &types.Issue{ + ID: "bd-b", + IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{DependsOnID: "bd-a", Type: types.DepBlocks}}, + } + expectDepBatchReads(mock, []driver.Value{"bd-b", "bd-a"}, []driver.Value{"bd-a", "bd-b"}) + mock.ExpectExec("INSERT INTO dependencies"). + WithArgs(depid.New("bd-a", "bd-b"), "bd-a", "bd-b", types.DepBlocks, "tester", sqlmock.AnyArg(), "{}", ""). + WillReturnResult(sqlmock.NewResult(0, 1)) + + var skipped []string + if _, err := PersistDependenciesWithOptionsResult(ctx, tx, []*types.Issue{a, b}, "tester", tolerant(&skipped)); err != nil { + t.Fatalf("error = %v, want the refusal skipped", err) + } + if want := "bd-b -> bd-a: " + domain.ErrDependencyCycle.Error(); len(skipped) != 1 || skipped[0] != want { + t.Fatalf("skipped = %#v, want [%q]", skipped, want) + } + finish(t, mock, tx) + }) +} + +// expectDepBatchReads expects newDepBatchLookups' reads for a batch with no +// wisps: the wisps probe, the presence of issueTargets in issues, and the edges +// of ids out of (issue_id) and into (each target column) both edge tables. +// The outgoing read of dependencies returns the stored rows. +func expectDepBatchReads(mock sqlmock.Sqlmock, issueTargets, ids []driver.Value, stored ...[]driver.Value) { + in := func(n int) string { return strings.TrimSuffix(strings.Repeat("?,", n), ",") } + mock.ExpectQuery(regexp.QuoteMeta("SELECT 1 FROM wisps LIMIT 1")).WillReturnError(sql.ErrNoRows) + present := sqlmock.NewRows([]string{"id"}) + for _, id := range issueTargets { + present.AddRow(id) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT id FROM issues WHERE id IN (" + in(len(issueTargets)) + ")")). + WithArgs(issueTargets...). + WillReturnRows(present) + for _, table := range []string{"dependencies", "wisp_dependencies"} { + rows := sqlmock.NewRows([]string{"issue_id", "target", "type"}) + if table == "dependencies" { + for _, row := range stored { + rows.AddRow(row...) + } + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT issue_id, " + DepTargetExpr + ", type FROM " + table + " WHERE issue_id IN (" + in(len(ids)) + ")")). + WithArgs(ids...). + WillReturnRows(rows) + } + for _, table := range []string{"dependencies", "wisp_dependencies"} { + for _, col := range []string{"depends_on_issue_id", "depends_on_wisp_id", "depends_on_external"} { + mock.ExpectQuery(regexp.QuoteMeta("SELECT issue_id, " + DepTargetExpr + ", type FROM " + table + " WHERE " + col + " IN (" + in(len(ids)) + ")")). + WithArgs(ids...). + WillReturnRows(sqlmock.NewRows([]string{"issue_id", "target", "type"})) + } + } +} From ae693ef7cca8e3bc3adf34295bb583270bc338cc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 17 Oct 2026 02:15:07 +0000 Subject: [PATCH 05/54] fix(http): bd gate create, add-waiter and discover over a connected HTTP workspace (#7621) * fix(http): gate create, add-waiter and discover through the library roles bd gate create, bd gate add-waiter and bd gate discover wrote through the legacy CreateIssue, AddDependency and map-based UpdateIssue, which the http store does not implement, so over a connected HTTP workspace each failed before any gate member reached the wire. They now run through the roles on every route (direct, proxied server, http): - gate create is ONE Lifecycle.Create carrying the gate members and a reverse blocks edge from the blocked issue to the new gate: atomic on every route, one createIssue over http behind issues.create.gates. The proxied-server route's private unit of work is gone; it uses the same role. - gate add-waiter reads through Reader.Get, deduplicates, and writes the whole waiter list through Lifecycle.Update guarded by the row version it read, retrying up to three times on ErrVersionMismatch, so concurrent adders cannot lose each other's waiter. Over http: issues.update.gates. - gate discover lists through issueops.ListGates and records the run ID through the same Lifecycle await_id patch gate check makes; bd close's gh:run persist (updateGateAwaitID) takes the same path. - Against a server without the token, each refuses before any write, naming the token. - gate discover skips template gates, as gate check already does, instead of recording a run ID on a proto every later pour copies. Tests: unit tests for the one-Create shape, the refusal, the add-waiter CAS retry, and discover over ListGates with its template skip; backend/http TestE2E_GatesOverHTTP (advertised and masked); four served corpus shapes, pending until the reference server advertises the gate tokens (verified to pass against a profile that does). CHANGELOG. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(gate): add-waiter backs off under contention; discover fails on a failed write Review follow-ups to the gate CLI port. - gate add-waiter's guarded write retried three times with no wait, so a convoy of workers registering on one gate at once failed all but a few (the proxied route's old unit of work had served them all). Each retry now waits a jittered, doubling backoff (10 ms to 400 ms, drawn from [d/2, d)), for at most 20 writes. The bound is attempts, not wall time: under a loaded server one round takes seconds, and a wall-clock budget failed the convoy after two or three rounds. A cancelled wait stops the loop. TestProxiedServerGateAddWaiterConvoy (direct SQL and proxied, real Dolt server) and TestE2E_GatesOverHTTP's concurrent case run 11 simultaneous adders (one duplicate): all exit 0, every waiter present once. - gate discover counts only the gates whose run ID was recorded and exits non-zero when any matched gate's write failed; it used to print the refused gate as updated and exit 0. TestE2E_GatesOverHTTP covers discover with and without issues.update.gates (fake gh). - TestServedGateCheckOverAwaitTypedGates masks issues.update.gates itself, so upgrading reference_server.json does not turn its gh:run case into a different test. - The discover listing test asserts closed gates are not listed. - CHANGELOG: discover's template skip and add-waiter's template refusal are behavior changes on the local routes; gate create refuses a negative --timeout; gate create/add-waiter follow dolt.auto-commit and use the role's commit messages; the proxied create no longer journals dependency_added. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 1 + CHANGELOG.md | 53 +++ backend/http/BUILD.bazel | 1 + backend/http/corpus/gate_check_served_test.go | 20 +- backend/http/corpus/testdata/corpus.yaml | 39 +- backend/http/e2e_gates_test.go | 315 ++++++++++++++++ cmd/bd/gate.go | 233 +++++++++--- cmd/bd/gate_discover.go | 55 ++- .../gate_frontdoor_parity_integration_test.go | 75 ++++ cmd/bd/gate_proxied_server.go | 146 -------- cmd/bd/gate_test.go | 353 ++++++++++++++++++ 11 files changed, 1064 insertions(+), 227 deletions(-) create mode 100644 backend/http/e2e_gates_test.go diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 33611719b..7c60e35ad 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -259,6 +259,7 @@ 34 31 TestProxiedServerComment 34 31 TestProxiedServerConfigB 34 31 TestProxiedServerEventsJournal +34 31 TestProxiedServerGateAddWaiterConvoy 34 31 TestProxiedServerIfRevisionGuardClose 34 31 TestProxiedServerMolBond diff --git a/CHANGELOG.md b/CHANGELOG.md index 1808f549a..10a63b334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -377,6 +377,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **`bd gate discover` skips template gates, as `bd gate check` does.** A + template gh:run gate is a proto that work is poured from; on the direct, + proxied and embedded routes `bd gate discover` used to record a discovered + run ID on it, which every later pour copied. It is now left alone on every + route. `bd gate add-waiter` on a template gate is refused by the update + role's template guard, where the direct, proxied and embedded routes all + used to write the waiter. +- **`bd gate create` refuses a negative `--timeout` on every route** + (`timeout cannot be negative`); the direct, proxied and embedded routes + used to store it. A `--type` longer than 32 characters fails with the same + typed validation error instead of a raw Dolt column error. +- **`bd gate create` and `bd gate add-waiter` follow `dolt.auto-commit` + like every other role write.** Under `dolt.auto-commit=off` `bd gate + create` no longer makes its own Dolt commit on the direct and embedded + routes (the proxied route already did not). The Dolt commit messages are + the role's: `bd: create gate X blocking Y` becomes the create's default + message, and `bd gate add-waiter`'s becomes the update's (`bd: update X` + on the direct and embedded routes, `update issue` on the proxied route). + On the proxied route `bd gate create` no longer journals a + `dependency_added` event on the blocked issue, matching the direct route + and `bd create --deps`. - **Gate members are bounds-checked in the library, on every local backend.** `issueops.Lifecycle.Create`, `Lifecycle.Update` and `BatchApplier` (create and update items) now refuse, with a typed `*issueops.GateFieldError` @@ -595,6 +616,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd gate create`, `bd gate add-waiter` and `bd gate discover` work over a + connected HTTP workspace.** They wrote through the legacy `CreateIssue`, + `AddDependency` and map-based `UpdateIssue`, which the HTTP store does not + implement, so each failed before any gate member reached the wire. They now + run through the library roles on every route: + - `bd gate create` is ONE `Lifecycle.Create` carrying the gate members and + a reverse `blocks` edge from the blocked issue to the new gate, so the + gate and its edge land atomically (one `createIssue` over HTTP, behind + `issues.create.gates`). The direct route used to make two writes and an + explicit commit; the proxied-server route now uses the same role. + - `bd gate add-waiter` reads the gate through `Reader.Get`, deduplicates, + and writes the whole waiter list through `Lifecycle.Update` guarded by + the row version it read. On a version mismatch it re-reads and retries + after a jittered, doubling backoff (10 ms up to 400 ms per wait, at most + 20 writes), so a convoy of workers registering on one gate at once all + succeed. On the direct route concurrent adders used to lose + each other's waiters while every one exited 0; no waiter is lost now. + Over HTTP it needs `issues.update.gates`. + - `bd gate discover` lists through `issueops.ListGates` (as `bd gate check` + does) and records each discovered run ID through `Lifecycle.Update`, the + same `await_id` write `bd gate check` and `bd close` make. Over HTTP it + needs `issues.update.gates`. Its summary now counts only the gates whose + run ID was recorded, and it exits non-zero when any matched gate's write + failed (it used to report such a gate as updated and exit 0). + + Against a server that does not advertise the token it needs, `bd gate + create` and `bd gate add-waiter` are refused before any write, naming the + token; `bd gate discover` writes nothing and reports the token as that + gate's error. The served corpus gains `mc-gate-create-timer`, + `mc-gate-check-timer-resolves`, `mc-gate-add-waiter` and + `mc-update-await-id` (pending until the reference server advertises the + gate tokens), pinned passing by `TestE2E_GatesOverHTTP`. - **`bd import` no longer reports a failed dependency check as a skipped dependency.** Import skips an edge the dependency checks refuse (a self-dependency, a cycle, or a blocker that is the issue's own ancestor or diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index 133286b71..da2d2454d 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -33,6 +33,7 @@ go_test( "attach_test.go", "e2e_cli_test.go", "e2e_export_test.go", + "e2e_gates_test.go", "e2e_heartbeat_test.go", "e2e_list_filters_test.go", "e2e_list_page_edges_test.go", diff --git a/backend/http/corpus/gate_check_served_test.go b/backend/http/corpus/gate_check_served_test.go index ffcc44b98..60d9f82c0 100644 --- a/backend/http/corpus/gate_check_served_test.go +++ b/backend/http/corpus/gate_check_served_test.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -51,11 +52,12 @@ func gateCheckWrites(reqs []string) []string { } // TestServedGateCheckOverAwaitTypedGates runs `bd gate check` over HTTP, through -// the reference proxy, against REAL await-typed gates. The reference server -// advertises neither gate token (issues.create.gates, issues.update.gates), so -// no create through it carries await_*: the gates are seeded straight into the -// served store before it listens, and every outcome is read back from that -// store: +// the reference proxy, against REAL await-typed gates, on a server that +// advertises neither gate token (issues.create.gates, issues.update.gates; the +// profile is masked here, so this holds whatever reference_server.json +// advertises). No create through it carries await_*: the gates are seeded +// straight into the served store before it listens, and every outcome is read +// back from that store: // // - --dry-run writes nothing; // - an expired timer closes, a pending one stays open; @@ -141,7 +143,13 @@ func TestServedGateCheckOverAwaitTypedGates(t *testing.T) { } }) - proxy := newReferenceProxy(t, loadReferenceProfile(t), srv.Addr()) + // The gh:run case pins a server that cannot take an await_id patch, so the + // profile is held without issues.update.gates whatever + // reference_server.json advertises: upgrading the reference profile must + // not turn this case into a different test. + profile := loadReferenceProfile(t) + profile.Capabilities = slices.DeleteFunc(profile.Capabilities, func(c string) bool { return c == "issues.update.gates" }) + proxy := newReferenceProxy(t, profile, srv.Addr()) reqs := &gateCheckRequestLog{next: proxy} proxyAddr := startProxy(t, reqs) diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index ba69ac9ee..668b72491 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -103,7 +103,11 @@ files: {"key": "t-ro", "id": "e2e-t-ro", "title": "target reopen", "type": "task"}, {"key": "t-dr", "id": "e2e-t-dr", "title": "target dep remove", "type": "task", "deps": [{"target": "t-db"}]}, {"key": "t-dc", "id": "e2e-t-dc", "title": "cascade root", "type": "epic"}, - {"key": "t-dc.1", "id": "e2e-t-dc.1", "title": "cascade child", "type": "task", "parent_key": "t-dc"} + {"key": "t-dc.1", "id": "e2e-t-dc.1", "title": "cascade child", "type": "task", "parent_key": "t-dc"}, + {"key": "t-gcr", "id": "e2e-t-gcr", "title": "target gate create", "type": "task"}, + {"key": "t-gct", "id": "e2e-t-gct", "title": "target gate check", "type": "task"}, + {"key": "t-gw", "id": "e2e-t-gw", "title": "target gate waiter", "type": "gate"}, + {"key": "t-gaw", "id": "e2e-t-gaw", "title": "target gate await id", "type": "gate"} ]} - name: bulk-ready-a.json bulk: {id_prefix: e2e-bra, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} @@ -445,7 +449,8 @@ shapes: source: core pack orders (gate list --all) args: [gate, list, --all, --json] # gate check lists the open gates through the same ListGates entry. No - # seeded gate carries an await type (no wire create accepts await_*), so the + # seeded gate carries an await type (the seed graph carries no gate members; + # mc-gate-check-timer-resolves below creates a real timer gate), so the # typed sweeps find nothing of their type and the untyped one lists the open # gates and evaluates none: the listing is what these shapes prove served, # and nothing resolves, escalates (no gt exec) or writes. @@ -796,3 +801,33 @@ shapes: expect_exit: 1 expect_stderr: 'unknown command "meta"' compare: exit + # ---------------------------------------------------------------- gates + # The gate writes: create (one createIssue carrying the gate members and the + # blocks edge), a real timer gate resolving under gate check, add-waiter (a + # guarded waiters patch) and update --await-id. A --timeout=1ns timer is + # expired by the time gate check runs, so the resolution is deterministic. + - id: mc-gate-create-timer + source: core formulas / gate-sweep (bd gate create --type=timer --timeout --blocks) + args: [gate, create, --type=timer, --timeout=1ns, --blocks, e2e-t-gcr, --json] + ignore_fields: [id] # each side mints its own gate id + pending: {slice: "SRV-1", reason: "bd gate create sends the gate members on createIssue behind issues.create.gates, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_GatesOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: mc-gate-check-timer-resolves + source: core gate-sweep.sh (gate check --type=timer --escalate) over a real timer gate + setup: [[gate, create, --type=timer, --timeout=1ns, --blocks, e2e-t-gct]] + args: [gate, check, --type=timer, --escalate] + compare: exit + verify: [[ready, --json, --limit, "0"]] # e2e-t-gct is ready on both sides + ignore_fields: [depends_on_id] # its (closed) gate's id is minted per workspace + pending: {slice: "SRV-1", reason: "the setup bd gate create sends the gate members on createIssue behind issues.create.gates, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_GatesOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: mc-gate-add-waiter + source: bd done --phase-complete (gate add-waiter) + args: [gate, add-waiter, e2e-t-gw, "rig/workers/a"] + compare: text + verify: [[show, --json, e2e-t-gw]] + pending: {slice: "SRV-1", reason: "bd gate add-waiter sends waiters on updateIssue behind issues.update.gates, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_GatesOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: mc-update-await-id + source: gate discover / gate check gh:run persist (update --await-id) + args: [update, e2e-t-gaw, --await-id, "12345", --json] + compare: exit + verify: [[show, --json, e2e-t-gaw]] + pending: {slice: "SRV-1", reason: "bd update --await-id sends await_id on updateIssue behind issues.update.gates, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_GatesOverHTTP); remove the mark when reference_server.json is upgraded"} diff --git a/backend/http/e2e_gates_test.go b/backend/http/e2e_gates_test.go new file mode 100644 index 000000000..d1ca36fe8 --- /dev/null +++ b/backend/http/e2e_gates_test.go @@ -0,0 +1,315 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" +) + +// gateTokens are the two behavior tokens the gate commands depend on: +// issues.create.gates for a create carrying gate members (bd gate create), +// issues.update.gates for an await_id or waiters patch (bd gate add-waiter, +// bd gate discover, gate check's gh:run persist, bd update --await-id). +var gateTokens = []string{"issues.create.gates", "issues.update.gates"} + +// withoutGateTokens rewrites the handshake into a server that predates the +// gate members: the live capability list minus the two gate tokens. +func withoutGateTokens(body map[string]any) { + caps, _ := body["capabilities"].([]any) + kept := make([]any, 0, len(caps)) + for _, c := range caps { + if s, _ := c.(string); slices.Contains(gateTokens, s) { + continue + } + kept = append(kept, c) + } + body["capabilities"] = kept +} + +// gateShow is the subset of `bd show --json` the gate cases read. +type gateShow struct { + ID string `json:"id"` + Status string `json:"status"` + IssueType string `json:"issue_type"` + AwaitType string `json:"await_type"` + AwaitID string `json:"await_id"` + Timeout int64 `json:"timeout"` + Waiters []string `json:"waiters"` +} + +func showGate(t *testing.T, f *remoteRolesFixture, id string) gateShow { + t.Helper() + out := strings.TrimSpace(f.mustRun(nil, "show", id, "--json").stdout) + var rows []gateShow + if err := json.Unmarshal([]byte(out), &rows); err != nil { + var one gateShow + if err1 := json.Unmarshal([]byte(out), &one); err1 != nil { + t.Fatalf("parse bd show --json: %v\n%s", err, out) + } + return one + } + if len(rows) == 0 { + t.Fatalf("bd show %s --json returned no row", id) + } + return rows[0] +} + +func readyIDs(t *testing.T, f *remoteRolesFixture) map[string]bool { + t.Helper() + ids := map[string]bool{} + for _, row := range jsonRows(t, f.mustRun(nil, "ready", "--json", "--limit", "0").stdout) { + if id, ok := row["id"].(string); ok { + ids[id] = true + } + } + return ids +} + +func writesOf(paths []string) []string { + var writes []string + for _, p := range paths { + if !strings.HasPrefix(p, http.MethodGet+" ") { + writes = append(writes, p) + } + } + return writes +} + +// TestE2E_GatesOverHTTP runs the gate commands through the real `bd` binary +// against an in-process `bd serve`. +// +// Against a server that advertises the gate tokens: +// - `bd gate create` is ONE createIssue carrying the gate members and the +// blocks edge (timer, gh:run and bead gates), and the blocked issue +// leaves the ready set; +// - `bd gate check --type=timer` closes an expired timer gate and the +// blocked issue becomes ready; +// - `bd gate add-waiter` registers a waiter once (the repeat writes nothing); +// - `bd update --await-id` sets await_id. +// +// Against a server that does not advertise them, each command is refused +// before any write, naming the token it needs, and nothing changes. +func TestE2E_GatesOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + + proxyAddr, counter := startCountingProxy(t, addr) + f := newHTTPFixture(t, bin, proxyAddr) + + var waiterGate string + + t.Run("advertised", func(t *testing.T) { + timerTarget := f.create("timer target") + counter.reset() + created := f.mustRun(nil, "gate", "create", "--type=timer", "--timeout=1ns", "--blocks", timerTarget, "--json") + if w := writesOf(counter.snapshot()); !slices.Equal(w, []string{"POST /v0/beads/issues"}) { + t.Errorf("gate create wrote %v, want exactly one createIssue", w) + } + timerGate := firstJSONField(t, created.stdout, "id") + if g := showGate(t, f, timerGate); g.IssueType != "gate" || g.AwaitType != "timer" || g.Timeout != 1 || g.Status != "open" { + t.Errorf("timer gate reads back %+v, want an open timer gate with a 1ns timeout", g) + } + if readyIDs(t, f)[timerTarget] { + t.Errorf("%s is ready while its gate is open", timerTarget) + } + + time.Sleep(10 * time.Millisecond) + r := f.mustRun(nil, "gate", "check", "--type=timer", "--escalate") + if !strings.Contains(r.stdout, "1 resolved") { + t.Errorf("gate check --type=timer = %q, want one resolved gate", r.stdout) + } + if g := showGate(t, f, timerGate); g.Status != "closed" { + t.Errorf("expired timer gate %s is %s after gate check, want closed", timerGate, g.Status) + } + if !readyIDs(t, f)[timerTarget] { + t.Errorf("%s is not ready after its timer gate resolved", timerTarget) + } + + runTarget := f.create("run target") + runGate := firstJSONField(t, f.mustRun(nil, "gate", "create", "--type=gh:run", "--await-id=ci-wf", "--repo=owner/repo", "--blocks", runTarget, "--json").stdout, "id") + if g := showGate(t, f, runGate); g.AwaitType != "gh:run" || g.AwaitID != "ci-wf" { + t.Errorf("gh:run gate reads back %+v, want await gh:run ci-wf", g) + } + + awaited := f.create("awaited bead") + beadTarget := f.create("bead target") + beadGate := firstJSONField(t, f.mustRun(nil, "gate", "create", "--type=bead", "--await-id", awaited, "--blocks", beadTarget, "--json").stdout, "id") + if g := showGate(t, f, beadGate); g.AwaitType != "bead" || g.AwaitID != awaited { + t.Errorf("bead gate reads back %+v, want await bead %s", g, awaited) + } + waiterGate = beadGate + + counter.reset() + r = f.mustRun(nil, "gate", "add-waiter", beadGate, "rig/workers/a") + if !strings.Contains(r.stdout, "Added waiter to gate "+beadGate+": rig/workers/a") { + t.Errorf("add-waiter output = %q", r.stdout) + } + if w := writesOf(counter.snapshot()); len(w) != 1 || !strings.HasPrefix(w[0], "PATCH /v0/beads/issues/") { + t.Errorf("add-waiter wrote %v, want one updateIssue", w) + } + counter.reset() + r = f.mustRun(nil, "gate", "add-waiter", beadGate, "rig/workers/a") + if !strings.Contains(r.stdout, "Waiter already registered on gate "+beadGate) { + t.Errorf("repeat add-waiter output = %q", r.stdout) + } + if w := writesOf(counter.snapshot()); len(w) != 0 { + t.Errorf("a repeat add-waiter wrote %v", w) + } + f.mustRun(nil, "gate", "add-waiter", beadGate, "rig/workers/b") + if g := showGate(t, f, beadGate); !slices.Equal(g.Waiters, []string{"rig/workers/a", "rig/workers/b"}) { + t.Errorf("waiters = %v, want [rig/workers/a rig/workers/b]", g.Waiters) + } + + f.mustRun(nil, "update", runGate, "--await-id", "12345", "--json") + if g := showGate(t, f, runGate); g.AwaitID != "12345" { + t.Errorf("await_id after update --await-id = %q, want 12345", g.AwaitID) + } + }) + + // A convoy registering on one gate at once: every adder must exit 0 and + // every waiter must be on the gate. Each round of simultaneous guarded + // writes has one winner, so this pins the retry backoff, not just the + // compare-and-set. + t.Run("concurrent add-waiter", func(t *testing.T) { + const adders = 10 + target := f.create("convoy target") + gate := firstJSONField(t, f.mustRun(nil, "gate", "create", "--blocks", target, "--json").stdout, "id") + results := make([]bdResult, adders+1) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i <= adders; i++ { + waiter := fmt.Sprintf("rig/workers/c%d", i%adders) // the last one repeats c0 + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + results[i] = f.run(nil, "gate", "add-waiter", gate, waiter) + }(i) + } + close(start) + wg.Wait() + for i, r := range results { + if r.code != 0 { + t.Errorf("adder %d exited %d: %s", i, r.code, r.stderr) + } + } + got := showGate(t, f, gate).Waiters + slices.Sort(got) + var want []string + for i := 0; i < adders; i++ { + want = append(want, fmt.Sprintf("rig/workers/c%d", i)) + } + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("waiters after %d concurrent adders = %v, want each exactly once: %v", adders+1, got, want) + } + }) + + // gate discover over HTTP: a fake gh answers the run listing, and the + // discovered run ID is recorded through updateIssue (issues.update.gates). + binDir := t.TempDir() + gh := `#!/bin/sh +case "$1 $2" in +"run list") echo '[{"databaseId":4242,"name":"ci-wf","status":"in_progress","conclusion":"","createdAt":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","workflowName":"ci-wf","headBranch":"main"}]' ;; +*) echo "unexpected gh $*" >&2; exit 3 ;; +esac +` + if err := os.WriteFile(filepath.Join(binDir, "gh"), []byte(gh), 0o755); err != nil { + t.Fatalf("write fake gh: %v", err) + } + ghEnv := []string{"PATH=" + binDir + string(os.PathListSeparator) + os.Getenv("PATH")} + + t.Run("discover", func(t *testing.T) { + target := f.create("discover target") + gate := firstJSONField(t, f.mustRun(nil, "gate", "create", "--type=gh:run", "--await-id=ci-wf", "--blocks", target, "--json").stdout, "id") + counter.reset() + r := f.run(ghEnv, "gate", "discover") + if r.code != 0 { + t.Fatalf("gate discover exited %d: %s %s", r.code, r.stdout, r.stderr) + } + if !strings.Contains(r.stdout, "Updated 1 gate(s)") { + t.Errorf("gate discover output = %q, want one updated gate", r.stdout) + } + if w := writesOf(counter.snapshot()); len(w) != 1 || !strings.HasPrefix(w[0], "PATCH /v0/beads/issues/") { + t.Errorf("gate discover wrote %v, want one updateIssue", w) + } + if g := showGate(t, f, gate); g.AwaitID != "4242" { + t.Errorf("await_id after discover = %q, want 4242", g.AwaitID) + } + }) + + t.Run("refused without the tokens", func(t *testing.T) { + if waiterGate == "" { + t.Skip("the advertised case created no gate") + } + maskedAddr, maskedCounter := startMaskingProxy(t, addr, withoutGateTokens) + m := newHTTPFixture(t, bin, maskedAddr) + target := m.create("masked target") + before := showGate(t, f, waiterGate) + + cases := []struct { + name string + args []string + token string + }{ + {"gate create", []string{"gate", "create", "--type=timer", "--timeout=1h", "--blocks", target}, "issues.create.gates"}, + {"gate create human", []string{"gate", "create", "--blocks", target}, "issues.create.gates"}, + {"gate add-waiter", []string{"gate", "add-waiter", waiterGate, "rig/workers/c"}, "issues.update.gates"}, + {"update --await-id", []string{"update", waiterGate, "--await-id", "999"}, "issues.update.gates"}, + } + for _, tc := range cases { + maskedCounter.reset() + r := m.run(nil, tc.args...) + if r.code == 0 { + t.Errorf("%s against a server without %s exited 0: %s", tc.name, tc.token, r.stdout) + } + if !strings.Contains(r.stderr, tc.token) { + t.Errorf("%s refusal does not name %s: %s", tc.name, tc.token, r.stderr) + } + if w := writesOf(maskedCounter.snapshot()); len(w) != 0 { + t.Errorf("a refused %s wrote %v; the refusal must come before any write", tc.name, w) + } + } + + // gate discover finds the run but cannot record it: the gate's error + // names the token, nothing is written, and the command fails rather + // than reporting the gate as updated. + runTarget := f.create("masked discover target") + runGate := firstJSONField(t, f.mustRun(nil, "gate", "create", "--type=gh:run", "--await-id=ci-wf", "--blocks", runTarget, "--json").stdout, "id") + maskedCounter.reset() + r := m.run(ghEnv, "gate", "discover") + if r.code == 0 { + t.Errorf("gate discover against a server without issues.update.gates exited 0: %s", r.stdout) + } + if !strings.Contains(r.stderr, "issues.update.gates") || !strings.Contains(r.stderr, runGate) { + t.Errorf("gate discover refusal does not name the gate and issues.update.gates: %s", r.stderr) + } + if strings.Contains(r.stdout, "Updated 1") { + t.Errorf("gate discover reported the refused gate as updated: %s", r.stdout) + } + if w := writesOf(maskedCounter.snapshot()); len(w) != 0 { + t.Errorf("a refused gate discover wrote %v", w) + } + if g := showGate(t, f, runGate); g.AwaitID != "ci-wf" { + t.Errorf("await_id after a refused discover = %q, want ci-wf untouched", g.AwaitID) + } + + if !readyIDs(t, m)[target] { + t.Errorf("%s left the ready set though every gate create was refused", target) + } + after := showGate(t, f, waiterGate) + if after.AwaitID != before.AwaitID || !slices.Equal(after.Waiters, before.Waiters) { + t.Errorf("refused writes changed the gate: before %+v after %+v", before, after) + } + }) +} diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 47c069ed9..25fe25e0a 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -6,8 +6,10 @@ import ( "encoding/json" "errors" "fmt" + "math/rand/v2" "os" "os/exec" + "slices" "strings" "time" @@ -237,9 +239,6 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return runGateAddWaiterProxiedServer(cmd, rootCtx, args) - } CheckReadonly("gate add-waiter") evt := metrics.NewCommandEvent("gate-add-waiter") @@ -251,34 +250,27 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio gateID := args[0] waiter := args[1] - ctx := rootCtx - var issue *types.Issue - var err error - - issue, err = store.GetIssue(ctx, gateID) + reader, err := openIssueReader() if err != nil { - return HandleError("gate not found: %s", gateID) + return HandleError("%v", err) + } + lifecycle, err := openIssueLifecycle() + if err != nil { + return HandleError("%v", err) + } + ctx, err := issueOpsContext(rootCtx) + if err != nil { + return HandleError("%v", err) } - if issue.IssueType != "gate" { - return HandleError("%s is not a gate issue (type=%s)", gateID, issue.IssueType) + added, err := addGateWaiter(ctx, reader, lifecycle, gateID, waiter) + if err != nil { + return HandleError("%v", err) } - - for _, w := range issue.Waiters { - if w == waiter { - renderGateWaiterAlready(gateID) - return nil - } - } - - newWaiters := append(issue.Waiters, waiter) - - updates := map[string]interface{}{ - "waiters": newWaiters, - } - if err := store.UpdateIssue(ctx, gateID, updates, currentActor()); err != nil { - return HandleError("updating gate: %v", err) + if !added { + renderGateWaiterAlready(gateID) + return nil } commandDidWrite.Store(true) @@ -288,6 +280,110 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio }, } +// addGateWaiter's compare-and-set budget. A version miss means another writer +// changed the gate between this command's read and its write; under a convoy +// (a phase finishing together, every worker registering on the same gate) each +// round of simultaneous writes has exactly one winner, so retrying at once +// would fail all but a handful of adders. Each retry therefore waits a +// jittered, doubling backoff first: gateAddWaiterBackoffBase, doubled per miss +// up to gateAddWaiterBackoffCap, each wait drawn uniformly from [d/2, d) so +// the losers of one round do not collide again in the next. +// +// The loop is bounded by gateAddWaiterMaxAttempts writes, not by wall time: +// one round's latency is the server's (a loaded remote server can take +// seconds per read and write), and a wall-clock budget would fail a convoy on +// a slow server after only two or three rounds. Each round has a winner, so +// the bound carries at least that many simultaneous adders; the jitter +// spreads them further. The waits themselves add up to at most about 6s, and +// a gate rewritten without pause still fails after the last attempt. +const ( + gateAddWaiterMaxAttempts = 20 + gateAddWaiterBackoffBase = 10 * time.Millisecond + gateAddWaiterBackoffCap = 400 * time.Millisecond +) + +// gateAddWaiterBackoff is the wait before retry number retry (1-based): +// jittered in [d/2, d) around d = base * 2^(retry-1), capped. +func gateAddWaiterBackoff(retry int) time.Duration { + d := gateAddWaiterBackoffBase + for i := 1; i < retry && d < gateAddWaiterBackoffCap; i++ { + d *= 2 + } + if d > gateAddWaiterBackoffCap { + d = gateAddWaiterBackoffCap + } + half := d / 2 + return half + time.Duration(rand.Int64N(int64(d-half))) // #nosec G404 -- retry jitter, not a secret +} + +// gateAddWaiterSleep waits d or until ctx is done. It is a variable so tests +// can record the backoff without sleeping. +var gateAddWaiterSleep = func(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// addGateWaiter registers waiter on the gate gateID through the Reader and +// Lifecycle roles, so every route (local, proxied server, and a connected HTTP +// workspace, which sends getIssue then updateIssue behind issues.update.gates) +// runs the same steps. It reports false, writing nothing, when the waiter is +// already registered. +// +// The write is a whole-value waiter list guarded by the row version the read +// returned, so two concurrent adders cannot lose each other's waiter: the +// second write misses its version, re-reads the list with the first waiter in +// it, and writes again. The deduplication is this command's, not the role's; +// the role stores the list it is given. +func addGateWaiter(ctx context.Context, reader issueops.Reader, lifecycle issueops.Lifecycle, gateID, waiter string) (bool, error) { + var lastErr error + attempts := 0 + for attempts < gateAddWaiterMaxAttempts { + if attempts > 0 { + if err := gateAddWaiterSleep(ctx, gateAddWaiterBackoff(attempts)); err != nil { + return false, fmt.Errorf("updating gate: %w", err) + } + } + attempts++ + details, err := reader.Get(ctx, issueops.GetRequest{ID: gateID}) + if errors.Is(err, storage.ErrNotFound) { + return false, fmt.Errorf("gate not found: %s", gateID) + } + if err != nil { + return false, fmt.Errorf("reading gate %s: %w", gateID, err) + } + gate := &details.Issue + if gate.IssueType != "gate" { + return false, fmt.Errorf("%s is not a gate issue (type=%s)", gateID, gate.IssueType) + } + if slices.Contains(gate.Waiters, waiter) { + return false, nil + } + + waiters := append(slices.Clone(gate.Waiters), waiter) + version := gate.RowVersion + _, err = lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: currentActor(), + IssueID: gate.ID, + Patch: issueops.IssuePatch{Waiters: issueops.Field[[]string]{Set: true, Value: waiters}}, + ExpectedVersion: &version, + }) + if err == nil { + return true, nil + } + if !errors.Is(err, issueops.ErrVersionMismatch) { + return false, fmt.Errorf("updating gate: %w", err) + } + lastErr = err + } + return false, fmt.Errorf("updating gate: %s changed on every one of %d attempts: %w", gateID, attempts, lastErr) +} + // renderGateWaiterAlready and renderGateWaiterAdded are shared by the direct // and proxied-server routes so `bd gate add-waiter` prints identically on both. func renderGateWaiterAlready(gateID string) { @@ -326,9 +422,6 @@ Examples: SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return runGateCreateProxiedServer(cmd, rootCtx) - } CheckReadonly("gate create") evt := metrics.NewCommandEvent("gate-create") @@ -343,37 +436,25 @@ Examples: return HandleErrorRespectJSON("%v", err) } - ctx := rootCtx - - targetIssue, err := store.GetIssue(ctx, in.blocksID) + reader, err := openIssueReader() if err != nil { - return HandleErrorRespectJSON("issue not found: %s", in.blocksID) + return HandleErrorRespectJSON("%v", err) + } + lifecycle, err := openIssueLifecycle() + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + ctx, err := issueOpsContext(rootCtx) + if err != nil { + return HandleErrorRespectJSON("%v", err) } - gate := buildGateIssue(in, targetIssue.ID) - metadata, metaErr := gateMetadataForCreate(in, targetIssue) - if metaErr != nil { - return HandleErrorRespectJSON("%v", metaErr) - } - gate.Metadata = metadata - - if err := store.CreateIssue(ctx, gate, currentActor()); err != nil { - return HandleErrorRespectJSON("creating gate: %v", err) + gate, targetIssue, err := createGateThroughRoles(ctx, reader, lifecycle, in) + if err != nil { + return HandleErrorRespectJSON("%v", err) } - dep := &types.Dependency{ - IssueID: targetIssue.ID, - DependsOnID: gate.ID, - Type: types.DepBlocks, - } - if err := store.AddDependency(ctx, dep, currentActor()); err != nil { - return HandleErrorRespectJSON("adding blocking dependency: %v", err) - } - - commitMsg := fmt.Sprintf("bd: create gate %s blocking %s", gate.ID, targetIssue.ID) - if err := store.Commit(ctx, commitMsg); err != nil && !isDoltNothingToCommit(err) { - return HandleErrorRespectJSON("failed to commit: %v", err) - } + commandDidWrite.Store(true) if jsonOutput { return outputJSON(gate) @@ -384,6 +465,48 @@ Examples: }, } +// createGateThroughRoles creates the gate and its blocking edge as ONE Lifecycle.Create: +// the gate row plus a reverse blocks dependency from the blocked issue to the +// new gate, in one transaction and one version commit on every route. A +// connected HTTP workspace sends it as one createIssue carrying the gate +// members, behind issues.create.gates; a server that does not advertise that +// token is refused before any request is sent, and nothing is written. +// +// It returns the created gate (without its dependency records, which +// `bd gate create` has never printed) and the blocked issue. +func createGateThroughRoles(ctx context.Context, reader issueops.Reader, lifecycle issueops.Lifecycle, in gateCreateInput) (*types.Issue, *types.Issue, error) { + details, err := reader.Get(ctx, issueops.GetRequest{ID: in.blocksID}) + if err != nil { + // Every target-lookup failure has always been reported as not-found. + return nil, nil, fmt.Errorf("issue not found: %s", in.blocksID) + } + target := &details.Issue + + gate := buildGateIssue(in, target.ID) + metadata, err := gateMetadataForCreate(in, target) + if err != nil { + return nil, nil, err + } + gate.Metadata = metadata + + result, err := lifecycle.Create(ctx, issueops.CreateRequest{ + Actor: currentActor(), + Issue: gate, + Dependencies: []issueops.CreateDependency{{ + TargetID: target.ID, + Type: types.DepBlocks, + Reverse: true, + }}, + }) + if err != nil { + return nil, nil, fmt.Errorf("creating gate: %w", err) + } + created := result.Issue + created.Dependencies = nil + created.Comments = nil + return created, target, nil +} + // gateCreateInput carries `bd gate create`'s parsed flags. Both routes gather // it through gatherGateCreateInput so they cannot drift on flag semantics. type gateCreateInput struct { diff --git a/cmd/bd/gate_discover.go b/cmd/bd/gate_discover.go index b1842e1de..1344107cb 100644 --- a/cmd/bd/gate_discover.go +++ b/cmd/bd/gate_discover.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) // GHWorkflowRun represents a GitHub workflow run from `gh run list --json` @@ -91,7 +92,7 @@ func runGateDiscover(cmd *cobra.Command, args []string) error { ctx := rootCtx - gates, err := findPendingGates() + gates, err := findPendingGates(ctx) if err != nil { return HandleError("finding gates: %v", err) } @@ -124,7 +125,11 @@ func runGateDiscover(cmd *cobra.Command, args []string) error { }) // Step 3/4: report matches (and, outside dry-run, persist them) + // matchCount counts the gates whose run ID was recorded (or, under + // --dry-run, would be); updateFailures the matched gates whose write + // failed, which fail the command after the summary. matchCount := 0 + updateFailures := 0 for _, m := range matches { if m.err != nil { fmt.Fprintf(os.Stderr, " %s %s - %v\n", @@ -140,20 +145,22 @@ func runGateDiscover(cmd *cobra.Command, args []string) error { continue } - matchCount++ runIDStr := strconv.FormatInt(m.run.DatabaseID, 10) if dryRun { + matchCount++ fmt.Printf(" %s %s → run %s (%s) [dry-run]\n", ui.RenderPass("✓"), ui.RenderID(m.gate.ID), runIDStr, m.run.Status) continue } if err := updateGateAwaitID(ctx, m.gate.ID, runIDStr); err != nil { + updateFailures++ fmt.Fprintf(os.Stderr, " %s %s - update failed: %v\n", ui.RenderFail("✗"), ui.RenderID(m.gate.ID), err) continue } + matchCount++ fmt.Printf(" %s %s → run %s (%s)\n", ui.RenderPass("✓"), ui.RenderID(m.gate.ID), runIDStr, m.run.Status) @@ -187,6 +194,9 @@ func runGateDiscover(cmd *cobra.Command, args []string) error { } return HandleError("querying GitHub runs failed for %d repo(s): %s", len(failures), strings.Join(details, "; ")) } + if updateFailures > 0 { + return HandleError("recording the discovered run ID failed for %d gate(s)", updateFailures) + } return nil } @@ -374,21 +384,32 @@ func workflowNameMatches(hint, workflowName, runName string) bool { // findPendingGates returns open gh:run gates that need run ID discovery. // This includes gates with empty AwaitID OR non-numeric AwaitID (workflow name hint). -func findPendingGates() ([]*types.Issue, error) { - var gates []*types.Issue - - gateType := types.IssueType("gate") - filter := types.IssueFilter{ - IssueType: &gateType, - ExcludeStatus: []types.Status{types.StatusClosed}, +// +// The open gates come from issueops.ListGates over the route's Querier, the +// entry `bd gate list` and `bd gate check` use, so a connected HTTP workspace +// answers it with queryIssues rather than refusing the raw storage filter. +func findPendingGates(ctx context.Context) ([]*types.Issue, error) { + querier, err := openQuerier() + if err != nil { + return nil, err } + return findPendingGatesWith(ctx, querier) +} - allGates, err := store.SearchIssues(rootCtx, "", filter) +// findPendingGatesWith is findPendingGates over a given Querier. Template +// gates are skipped: a template is read-only (work is poured from it), so its +// await_id is never a run to discover. +func findPendingGatesWith(ctx context.Context, querier issueops.Querier) ([]*types.Issue, error) { + allGates, err := issueops.ListGates(ctx, querier, issueops.GateListRequest{}) if err != nil { return nil, fmt.Errorf("search gates: %w", err) } + var gates []*types.Issue for _, g := range allGates { + if g.IsTemplate { + continue + } if needsDiscovery(g) { gates = append(gates, g) } @@ -576,13 +597,11 @@ func matchGateToRun(gate *types.Issue, runs []GHWorkflowRun, maxAge time.Duratio return nil } -// updateGateAwaitID updates a gate's await_id field +// updateGateAwaitID records a discovered run ID as a gate's await_id through +// the Lifecycle role (persistGateCheckAwaitID), the write `bd gate check` and +// `bd close` make for the same discovery. A connected HTTP workspace sends it +// as updateIssue behind issues.update.gates, and a server that does not +// advertise that token refuses it before any request is sent. func updateGateAwaitID(_ interface{}, gateID, runID string) error { - updates := map[string]interface{}{ - "await_id": runID, - } - if err := store.UpdateIssue(rootCtx, gateID, updates, currentActor()); err != nil { - return err - } - return nil + return persistGateCheckAwaitID(rootCtx, gateID, runID) } diff --git a/cmd/bd/gate_frontdoor_parity_integration_test.go b/cmd/bd/gate_frontdoor_parity_integration_test.go index 1f251755f..96317fc66 100644 --- a/cmd/bd/gate_frontdoor_parity_integration_test.go +++ b/cmd/bd/gate_frontdoor_parity_integration_test.go @@ -4,8 +4,11 @@ package main import ( "encoding/json" + "fmt" "os/exec" + "slices" "strings" + "sync" "testing" ) @@ -124,3 +127,75 @@ func TestProxiedServerGateFrontDoorDirectServerProxyParity(t *testing.T) { } } } + +// TestProxiedServerGateAddWaiterConvoy registers ten waiters (plus one +// duplicate) on one gate at once, on the direct SQL route and the proxied +// route against a real Dolt server: every adder must exit 0 and every waiter +// must be on the gate exactly once. add-waiter is a guarded whole-list write, +// so each round of simultaneous writes has one winner; this pins the retry's +// backoff and budget, which must carry a convoy finishing a phase together. +func TestProxiedServerGateAddWaiterConvoy(t *testing.T) { + requireSharedProxiedServer(t) + bd := buildEmbeddedBD(t) + directProject := newServerModeProject(t, bd, "cv") + proxiedProject := newSharedProxiedProject(t, bd, "cv") + modes := []gateFrontDoorRunner{ + {name: "direct-sql", dir: directProject.dir, env: func(string) []string { return directProject.env }}, + {name: "proxied", dir: proxiedProject.dir, env: bdProxiedEnv}, + } + const adders = 10 + for _, r := range modes { + r.mustRun(t, bd, "create", "Convoy target", "--id", "cv-target") + gateID := parseCreatedGateID(t, r.mustRun(t, bd, "gate", "create", "--blocks", "cv-target")) + + type result struct { + stdout, stderr string + err error + } + results := make([]result, adders+1) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i <= adders; i++ { + waiter := fmt.Sprintf("rig/workers/c%d", i%adders) // the last one repeats c0 + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + cmd := exec.Command(bd, "gate", "add-waiter", gateID, waiter) + cmd.Dir = r.dir + cmd.Env = r.env(r.dir) + var stdout, stderr strings.Builder + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + results[i] = result{stdout.String(), stderr.String(), err} + }(i) + } + close(start) + wg.Wait() + for i, res := range results { + if res.err != nil { + t.Errorf("[%s] adder %d failed: %v\nstderr:\n%s", r.name, i, res.err, res.stderr) + } + } + + var shown map[string]any + out := r.mustRun(t, bd, "gate", "show", gateID, "--json") + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &shown); err != nil { + t.Fatalf("[%s] parse gate show --json: %v\n%s", r.name, err, out) + } + raw, _ := shown["waiters"].([]any) + got := make([]string, 0, len(raw)) + for _, w := range raw { + got = append(got, fmt.Sprint(w)) + } + want := make([]string, 0, adders) + for i := 0; i < adders; i++ { + want = append(want, fmt.Sprintf("rig/workers/c%d", i)) + } + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("[%s] waiters after %d concurrent adders = %v, want each exactly once: %v", r.name, adders+1, got, want) + } + } +} diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index 15d5a133b..0821180c6 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -268,152 +268,6 @@ func runGateShowProxiedServer(_ *cobra.Command, ctx context.Context, args []stri return nil } -type gateAddWaiterApply struct { - already bool - after *types.Issue -} - -func runGateAddWaiterProxiedServer(_ *cobra.Command, ctx context.Context, args []string) error { - CheckReadonly("gate add-waiter") - - evt := metrics.NewCommandEvent("gate-add-waiter") - defer func() { - if c := metrics.Global(); c != nil { - c.CloseEventAndAdd(evt) - } - }() - - gateID := args[0] - waiter := args[1] - - if uowProvider == nil { - return HandleError("proxied-server UOW provider not initialized") - } - - applied, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (gateAddWaiterApply, string, error) { - var out gateAddWaiterApply - - issue, err := uw.IssueUseCase().GetIssue(ctx, gateID) - if gateProxiedNotFound(err) { - return out, "", fmt.Errorf("gate not found: %s", gateID) - } - if err != nil { - return out, "", fmt.Errorf("reading gate %s: %w", gateID, err) - } - if issue.IssueType != "gate" { - return out, "", fmt.Errorf("%s is not a gate issue (type=%s)", gateID, issue.IssueType) - } - - for _, w := range issue.Waiters { - if w == waiter { - out.already = true - // Empty commit message: a registered waiter is a no-op, and a - // no-op writes no Dolt commit. - return out, "", nil - } - } - - newWaiters := append(issue.Waiters, waiter) - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, currentActor()); err != nil { - return out, "", fmt.Errorf("updating gate: %w", err) - } - if after, getErr := uw.IssueUseCase().GetIssue(ctx, gateID); getErr == nil { - out.after = after - } - return out, fmt.Sprintf("bd: gate add-waiter %s", gateID), nil - }) - if err != nil { - return HandleError("%v", err) - } - - if applied.already { - renderGateWaiterAlready(gateID) - return nil - } - - commandDidWrite.Store(true) - - renderGateWaiterAdded(gateID, waiter) - return nil -} - -type gateCreateApply struct { - gate *types.Issue - target *types.Issue -} - -func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { - CheckReadonly("gate create") - - evt := metrics.NewCommandEvent("gate-create") - defer func() { - if c := metrics.Global(); c != nil { - c.CloseEventAndAdd(evt) - } - }() - - in, err := gatherGateCreateInput(cmd) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - if uowProvider == nil { - return HandleError("proxied-server UOW provider not initialized") - } - - // One transaction, one Dolt commit for the whole invocation: the direct - // route's create + add-dependency + explicit store.Commit collapse into a - // single unit of work carrying the same commit message. Semantically - // equivalent, minus the window where the gate exists without its edge. - applied, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (gateCreateApply, string, error) { - var out gateCreateApply - - target, err := uw.IssueUseCase().GetIssue(ctx, in.blocksID) - if err != nil { - // The direct route reports every target-lookup failure as - // not-found; keep that message for parity. - return out, "", fmt.Errorf("issue not found: %s", in.blocksID) - } - - gate := buildGateIssue(in, target.ID) - metadata, metaErr := gateMetadataForCreate(in, target) - if metaErr != nil { - return out, "", metaErr - } - gate.Metadata = metadata - - res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, currentActor()) - if err != nil { - return out, "", fmt.Errorf("creating gate: %w", err) - } - - dep := &types.Dependency{ - IssueID: target.ID, - DependsOnID: res.Issue.ID, - Type: types.DepBlocks, - } - if err := uw.DependencyUseCase().AddDependency(ctx, dep, currentActor()); err != nil { - return out, "", fmt.Errorf("adding blocking dependency: %w", err) - } - - out.gate = res.Issue - out.target = target - return out, fmt.Sprintf("bd: create gate %s blocking %s", res.Issue.ID, target.ID), nil - }) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - commandDidWrite.Store(true) - - if jsonOutput { - return outputJSON(applied.gate) - } - - renderGateCreated(applied.gate, applied.target, in) - return nil -} - type gateResolveApply struct { before *types.Issue after *types.Issue diff --git a/cmd/bd/gate_test.go b/cmd/bd/gate_test.go index 980906b9f..06bcee406 100644 --- a/cmd/bd/gate_test.go +++ b/cmd/bd/gate_test.go @@ -2505,3 +2505,356 @@ func TestGateCheckAwaitIDPassesOtherRefusalsThrough(t *testing.T) { }) } } + +// fakeGateRoles is the Reader and Lifecycle `bd gate create` and +// `bd gate add-waiter` run over. Its Update honors ExpectedVersion the way +// every backend does, and can simulate concurrent adders: each of the first +// `concurrent` updates finds that another writer appended its waiter (and +// reminted the row version) after the command's read. +type fakeGateRoles struct { + issueops.Lifecycle + issues map[string]*types.Issue + creates []issueops.CreateRequest + updates []issueops.UpdateRequest + createErr error + updateErr error + concurrent []string +} + +func newFakeGateRoles(issues ...*types.Issue) *fakeGateRoles { + f := &fakeGateRoles{issues: map[string]*types.Issue{}} + for _, issue := range issues { + f.issues[issue.ID] = issue + } + return f +} + +func (f *fakeGateRoles) Ready(context.Context, issueops.ReadyRequest) (issueops.IssuePage, error) { + panic("unused") +} + +func (f *fakeGateRoles) List(context.Context, issueops.ListRequest) (issueops.IssuePage, error) { + panic("unused") +} + +func (f *fakeGateRoles) Get(_ context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + issue, ok := f.issues[req.ID] + if !ok { + return nil, storage.ErrNotFound + } + copied := *issue + copied.Waiters = slices.Clone(issue.Waiters) + return &issueops.IssueDetails{Issue: copied}, nil +} + +func (f *fakeGateRoles) Create(_ context.Context, req issueops.CreateRequest) (issueops.CreateResult, error) { + f.creates = append(f.creates, req) + if f.createErr != nil { + return issueops.CreateResult{}, f.createErr + } + created := *req.Issue + created.ID = "bd-gate1" + created.Dependencies = []*types.Dependency{{IssueID: req.Dependencies[0].TargetID, DependsOnID: created.ID, Type: types.DepBlocks}} + f.issues[created.ID] = &created + return issueops.CreateResult{Issue: &created}, nil +} + +func (f *fakeGateRoles) Update(_ context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + f.updates = append(f.updates, req) + if f.updateErr != nil { + return issueops.UpdateResult{}, f.updateErr + } + issue := f.issues[req.IssueID] + if len(f.concurrent) > 0 { + issue.Waiters = append(issue.Waiters, f.concurrent[0]) + issue.RowVersion++ + f.concurrent = f.concurrent[1:] + } + if req.ExpectedVersion != nil && *req.ExpectedVersion != issue.RowVersion { + return issueops.UpdateResult{}, issueops.ErrVersionMismatch + } + if req.Patch.Waiters.Set { + issue.Waiters = slices.Clone(req.Patch.Waiters.Value) + } + issue.RowVersion++ + return issueops.UpdateResult{Issue: issue}, nil +} + +func setGateTestActor(t *testing.T) { + t.Helper() + origActor := actor + t.Cleanup(func() { actor = origActor }) + actor = "test-actor" +} + +// TestCreateGateIsOneCreateWithAReverseBlocksEdge: `bd gate create` is ONE +// Lifecycle.Create carrying the gate members and the blocks edge from the +// blocked issue to the new gate, so the gate and its edge land atomically on +// every route (a connected HTTP workspace sends one createIssue). +func TestCreateGateIsOneCreateWithAReverseBlocksEdge(t *testing.T) { + setGateTestActor(t) + roles := newFakeGateRoles(&types.Issue{ID: "bd-target", Title: "blocked", IssueType: types.TypeTask}) + + in := gateCreateInput{blocksID: "bd-target", gateType: "timer", timeout: 90 * time.Minute, reason: "cool down"} + gate, target, err := createGateThroughRoles(context.Background(), roles, roles, in) + if err != nil { + t.Fatalf("createGateThroughRoles: %v", err) + } + if len(roles.creates) != 1 || len(roles.updates) != 0 { + t.Fatalf("createGateThroughRoles made %d creates and %d updates, want exactly one create", len(roles.creates), len(roles.updates)) + } + req := roles.creates[0] + wantDeps := []issueops.CreateDependency{{TargetID: "bd-target", Type: types.DepBlocks, Reverse: true}} + if !reflect.DeepEqual(req.Dependencies, wantDeps) { + t.Fatalf("create dependencies = %+v, want %+v", req.Dependencies, wantDeps) + } + if req.Actor != "test-actor" { + t.Errorf("create actor = %q, want test-actor", req.Actor) + } + if req.Issue.IssueType != "gate" || req.Issue.AwaitType != "timer" || req.Issue.Timeout != 90*time.Minute { + t.Errorf("created gate = type %q await_type %q timeout %s, want a 90m timer gate", req.Issue.IssueType, req.Issue.AwaitType, req.Issue.Timeout) + } + if !strings.Contains(req.Issue.Description, "cool down") { + t.Errorf("gate description %q does not carry the reason", req.Issue.Description) + } + if gate.ID != "bd-gate1" || target.ID != "bd-target" { + t.Errorf("createGateThroughRoles returned gate %q target %q", gate.ID, target.ID) + } + if gate.Dependencies != nil { + t.Errorf("createGateThroughRoles returned the gate with dependency records %v; bd gate create never printed them", gate.Dependencies) + } +} + +// TestCreateGateRefusalWritesNothing: a refused create (a server that does not +// advertise issues.create.gates) surfaces the refusal naming the token, and a +// missing blocked issue never reaches the create. +func TestCreateGateRefusalWritesNothing(t *testing.T) { + setGateTestActor(t) + + t.Run("token refused", func(t *testing.T) { + roles := newFakeGateRoles(&types.Issue{ID: "bd-target", IssueType: types.TypeTask}) + roles.createErr = &storage.ErrUnsupported{Op: "createIssue", Backend: "http", Capability: "issues.create.gates"} + _, _, err := createGateThroughRoles(context.Background(), roles, roles, gateCreateInput{blocksID: "bd-target", gateType: "timer"}) + var unsupported *storage.ErrUnsupported + if !errors.As(err, &unsupported) || !strings.Contains(err.Error(), "issues.create.gates") { + t.Fatalf("createGateThroughRoles error = %v, want the refusal naming issues.create.gates", err) + } + if _, ok := roles.issues["bd-gate1"]; ok || len(roles.updates) != 0 { + t.Fatalf("a refused create wrote something") + } + }) + + t.Run("missing target", func(t *testing.T) { + roles := newFakeGateRoles() + _, _, err := createGateThroughRoles(context.Background(), roles, roles, gateCreateInput{blocksID: "bd-absent", gateType: "human"}) + if err == nil || err.Error() != "issue not found: bd-absent" { + t.Fatalf("createGateThroughRoles error = %v, want issue not found: bd-absent", err) + } + if len(roles.creates) != 0 { + t.Fatalf("createGateThroughRoles created a gate for a missing target") + } + }) +} + +// TestAddGateWaiterIsAGuardedWholeListUpdate: the waiter list is written whole, +// guarded by the version the read returned. A concurrent adder makes the write +// miss; the retry re-reads (keeping the other waiter) and re-deduplicates. +func TestAddGateWaiterIsAGuardedWholeListUpdate(t *testing.T) { + setGateTestActor(t) + origSleep := gateAddWaiterSleep + t.Cleanup(func() { gateAddWaiterSleep = origSleep }) + var sleeps []time.Duration + gateAddWaiterSleep = func(_ context.Context, d time.Duration) error { + sleeps = append(sleeps, d) + return nil + } + gate := func() *types.Issue { + return &types.Issue{ID: "bd-g", IssueType: "gate", Waiters: []string{"a"}, RowVersion: 7} + } + + t.Run("first write", func(t *testing.T) { + roles := newFakeGateRoles(gate()) + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added", added, err) + } + if len(roles.updates) != 1 { + t.Fatalf("updates = %d, want 1", len(roles.updates)) + } + req := roles.updates[0] + if req.ExpectedVersion == nil || *req.ExpectedVersion != 7 { + t.Errorf("update not guarded by the read's version 7: %v", req.ExpectedVersion) + } + want := issueops.IssuePatch{Waiters: issueops.Field[[]string]{Set: true, Value: []string{"a", "b"}}} + if !reflect.DeepEqual(req.Patch, want) { + t.Errorf("patch = %+v, want only the whole waiter list [a b]", req.Patch) + } + }) + + t.Run("concurrent adder retries", func(t *testing.T) { + roles := newFakeGateRoles(gate()) + roles.concurrent = []string{"x"} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added after one retry", added, err) + } + if len(roles.updates) != 2 { + t.Fatalf("updates = %d, want the missed write and its retry", len(roles.updates)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "x", "b"}) { + t.Errorf("waiters = %v, want [a x b]: the concurrent waiter must survive", got) + } + }) + + t.Run("concurrent adder of the same waiter", func(t *testing.T) { + roles := newFakeGateRoles(gate()) + roles.concurrent = []string{"b"} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || added { + t.Fatalf("addGateWaiter = %v, %v; want already registered after the re-read", added, err) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b]", got) + } + }) + + t.Run("already registered writes nothing", func(t *testing.T) { + roles := newFakeGateRoles(gate()) + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "a") + if err != nil || added || len(roles.updates) != 0 { + t.Fatalf("addGateWaiter = %v, %v with %d updates; want a no-op", added, err, len(roles.updates)) + } + }) + + t.Run("backs off between attempts", func(t *testing.T) { + sleeps = nil + roles := newFakeGateRoles(gate()) + roles.concurrent = []string{"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9"} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added once the nine concurrent writers are done", added, err) + } + if len(roles.updates) != 10 || len(sleeps) != 9 { + t.Fatalf("updates = %d, sleeps = %d; want 10 writes with a backoff before each retry", len(roles.updates), len(sleeps)) + } + want := gateAddWaiterBackoffBase + for i, d := range sleeps { + if d < want/2 || d >= want { + t.Errorf("backoff %d = %s, want in [%s, %s)", i+1, d, want/2, want) + } + if want*2 <= gateAddWaiterBackoffCap { + want *= 2 + } else { + want = gateAddWaiterBackoffCap + } + } + }) + + t.Run("gives up after the attempt bound", func(t *testing.T) { + sleeps = nil + roles := newFakeGateRoles(gate()) + for i := 0; i <= gateAddWaiterMaxAttempts; i++ { + roles.concurrent = append(roles.concurrent, fmt.Sprintf("x%d", i)) + } + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Fatalf("addGateWaiter error = %v, want the version mismatch", err) + } + if len(roles.updates) != gateAddWaiterMaxAttempts { + t.Fatalf("updates = %d, want %d attempts", len(roles.updates), gateAddWaiterMaxAttempts) + } + }) + + t.Run("a cancelled wait stops the loop", func(t *testing.T) { + roles := newFakeGateRoles(gate()) + roles.concurrent = []string{"x1", "x2"} + gateAddWaiterSleep = func(context.Context, time.Duration) error { return context.Canceled } + t.Cleanup(func() { + gateAddWaiterSleep = func(_ context.Context, d time.Duration) error { + sleeps = append(sleeps, d) + return nil + } + }) + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, context.Canceled) || len(roles.updates) != 1 { + t.Fatalf("addGateWaiter = %v after %d updates, want the cancellation after one write", err, len(roles.updates)) + } + }) + + t.Run("refusal is not retried", func(t *testing.T) { + roles := newFakeGateRoles(gate()) + roles.updateErr = &storage.ErrUnsupported{Op: "updateIssue", Backend: "http", Capability: "issues.update.gates"} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err == nil || !strings.Contains(err.Error(), "issues.update.gates") || len(roles.updates) != 1 { + t.Fatalf("addGateWaiter error = %v after %d updates, want one refused update naming issues.update.gates", err, len(roles.updates)) + } + }) + + t.Run("not a gate", func(t *testing.T) { + roles := newFakeGateRoles(&types.Issue{ID: "bd-t", IssueType: types.TypeTask}) + _, err := addGateWaiter(context.Background(), roles, roles, "bd-t", "b") + if err == nil || !strings.Contains(err.Error(), "is not a gate issue") { + t.Fatalf("addGateWaiter error = %v, want the not-a-gate refusal", err) + } + }) + + t.Run("missing gate", func(t *testing.T) { + roles := newFakeGateRoles() + _, err := addGateWaiter(context.Background(), roles, roles, "bd-none", "b") + if err == nil || err.Error() != "gate not found: bd-none" { + t.Fatalf("addGateWaiter error = %v, want gate not found: bd-none", err) + } + }) +} + +// TestFindPendingGatesListsThroughListGates: gate discover lists the open gates +// through issueops.ListGates (queryIssues over HTTP), never the raw storage +// filter, and skips templates and gates that already hold a run ID. +func TestFindPendingGatesListsThroughListGates(t *testing.T) { + fake := &fakeGateCheckStore{issues: []*types.Issue{ + {ID: "bd-hint", IssueType: "gate", AwaitType: "gh:run", AwaitID: "release.yml"}, + {ID: "bd-empty", IssueType: "gate", AwaitType: "gh:run"}, + {ID: "bd-numeric", IssueType: "gate", AwaitType: "gh:run", AwaitID: "12345"}, + {ID: "bd-template", IssueType: "gate", AwaitType: "gh:run", IsTemplate: true}, + {ID: "bd-timer", IssueType: "gate", AwaitType: "timer"}, + }} + gates, err := findPendingGatesWith(context.Background(), fake) + if err != nil { + t.Fatalf("findPendingGatesWith: %v", err) + } + var ids []string + for _, g := range gates { + ids = append(ids, g.ID) + } + if !reflect.DeepEqual(ids, []string{"bd-hint", "bd-empty"}) { + t.Fatalf("pending gates = %v, want [bd-hint bd-empty]", ids) + } + if fake.searchCalled { + t.Fatal("gate discover used the raw SearchIssues filter") + } + if len(fake.queries) != 1 || fake.queries[0].Expression != issueops.GateListExpression { + t.Fatalf("queries = %+v, want one ListGates query", fake.queries) + } + if fake.queries[0].IncludeClosed { + t.Fatal("gate discover listed closed gates; only open gates await a run") + } +} + +// TestUpdateGateAwaitIDGoesThroughTheLifecycleRole: gate discover's and +// bd close's await_id persist is the Lifecycle patch gate check makes, never +// the legacy map-based UpdateIssue the HTTP store does not serve. +func TestUpdateGateAwaitIDGoesThroughTheLifecycleRole(t *testing.T) { + origStore := store + t.Cleanup(func() { store = origStore }) + setGateTestActor(t) + fake := &fakeGateCheckStore{} + store = fake + + if err := updateGateAwaitID(nil, "bd-run", "4242"); err != nil { + t.Fatalf("updateGateAwaitID: %v", err) + } + want := issueops.IssuePatch{AwaitID: issueops.Field[string]{Set: true, Value: "4242"}} + if len(fake.updates) != 1 || fake.updates[0].IssueID != "bd-run" || !reflect.DeepEqual(fake.updates[0].Patch, want) { + t.Fatalf("updates = %+v, want one AwaitID=4242 patch on bd-run", fake.updates) + } +} From 925e26bc48374753902e51817872f80b753b6df1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 17 Oct 2026 06:00:33 +0000 Subject: [PATCH 06/54] fix(http): serve bd list --title-contains by folding it onto title (#7631) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(http): serve bd list --title-contains by folding it onto title `bd list --title-contains X` refused against an HTTP workspace because listIssues publishes no title_contains parameter. The store compiles TitleContains and TitleSearch to the same predicate (LOWER(title) LIKE '%v%'), so the client now folds TitleContains onto TitleSearch before the pre-dial gate (encode.FoldTitleContains) when TitleSearch is empty or equal. The request then rides the existing `title` parameter behind issues.list.title, and against a server without that token it refuses before dialing, naming the token. A --title and --title-contains that differ still refuse: the listing has one title parameter and dropping either value would widen the answer. No wire change. The ledger row E-ListRequest.TitleContains narrows to the differing pair. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(http): fold case-only title pairs and qualify the CHANGELOG Review fixups for #7631. - FoldTitleContains also folds a pair that differs only in letter case. The store binds each value as '%' + strings.ToLower(v) + '%', so a pair equal under strings.ToLower is one predicate. It does not use strings.EqualFold, which matches pairs such as "ſ" and "s" that lower to different patterns. Unit cases pin both directions, and the served test answers a case-only pair like the single flags. - CHANGELOG: name what still refuses over HTTP: a pair that differs other than in case, either flag with --parent in the default tree view, and --closed-after, which gascity's polecat pre-flight lookup of recently closed beads (step 3d) uses. - Corpus: add gc-polecat-title-contains-closed-after, the 3d lookup, pending until the reference server advertises issues.list.title and listIssues carries closed_after (E-ListRequest.ClosedAfter). - Drop two assertions that could not fail (a request passed by value cannot be mutated), and bring the e2e shape header, the tokenedListFilters comment, the encoder table comment and the E-ListRequest.TitleContains ledger row (doc regenerated) up to date. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 +++ backend/http/corpus/testdata/corpus.yaml | 8 +++ backend/http/e2e_list_filters_test.go | 32 ++++++++++- engdocs/design/http-divergence-ledger.md | 2 +- internal/httpclient/encode/BUILD.bazel | 2 + internal/httpclient/encode/ledger_fields.go | 3 +- internal/httpclient/encode/list_fold.go | 41 ++++++++++++++ internal/httpclient/encode/list_fold_test.go | 59 ++++++++++++++++++++ internal/httpclient/encode/table.go | 4 ++ internal/httpclient/list_scope_test.go | 24 ++++++++ internal/httpclient/role_reader.go | 11 +++- internal/httpclient/served_list_test.go | 41 +++++++++++++- 12 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 internal/httpclient/encode/list_fold.go create mode 100644 internal/httpclient/encode/list_fold_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 10a63b334..9f1809fd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -616,6 +616,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `bd list --title-contains X` works against an HTTP workspace. The client + folds it onto the listing's existing `title` parameter (the store compiles + both flags to the same case-insensitive substring predicate), so it needs + the `issues.list.title` capability, like `--title`. Over HTTP these still + refuse: a `--title` and a `--title-contains` that differ other than in + letter case (the listing has one title parameter, and dropping either value + would widen the answer); either flag with `--parent` in the default tree + view; and `--closed-after`, so gascity's polecat pre-flight lookup of + recently closed beads still fails. - **`bd gate create`, `bd gate add-waiter` and `bd gate discover` work over a connected HTTP workspace.** They wrote through the legacy `CreateIssue`, `AddDependency` and map-based `UpdateIssue`, which the HTTP store does not diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 668b72491..ad989aa79 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -308,6 +308,14 @@ shapes: source: mc skills/pr-review (list --all --id --json --limit) args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] pending: {slice: "SRV-1", reason: "S15b sends --id as listIssues id behind issues.list.ids, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListScopeFiltersOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: gc-polecat-title-contains + source: gascity core formulas/mol-polecat-base.toml (list --title-contains --status a,b,c --json) + args: [list, --title-contains, "Routed", --status, "open,in_progress,blocked", --json] + pending: {slice: "SRV-1", reason: "--title-contains folds onto listIssues title behind issues.list.title, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListScopeFiltersOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: gc-polecat-title-contains-closed-after + source: gascity core formulas/mol-polecat-base.toml (pre-flight 3d, list --title-contains --status closed --closed-after DATE --json) + args: [list, --title-contains, "closed task", --status, closed, --closed-after, "2000-01-01", --json] + pending: {slice: "SRV-1+list-closed-after", reason: "two refusals: against the reference server --title-contains folds onto listIssues title behind issues.list.title, which it does not advertise, so the client refuses before dialing; past that, listIssues publishes no closed_after parameter, so the encoder refuses --closed-after (E-ListRequest.ClosedAfter); remove the mark when reference_server.json is upgraded and listIssues carries closed_after"} - id: mc-list-assignee-status source: mc review summaries (list --assignee --json --limit --status) args: [list, --assignee, alice, --json, --limit, "10", --status, open] diff --git a/backend/http/e2e_list_filters_test.go b/backend/http/e2e_list_filters_test.go index a25398e0e..cd26c97ec 100644 --- a/backend/http/e2e_list_filters_test.go +++ b/backend/http/e2e_list_filters_test.go @@ -8,12 +8,16 @@ import ( "testing" ) -// listScopeShapes are the four maintainer-city `bd list` shapes S15b serves, -// one per corpus shape (backend/http/corpus/testdata/corpus.yaml), with the -// token each needs and the titles each must answer on the scope seed: +// listScopeShapes are the tokened `bd list` shapes, with the token each needs +// and the titles each must answer on the scope seed. Five are corpus shapes +// (backend/http/corpus/testdata/corpus.yaml) and one pins the title fold: // - gc-list-on-boot: gc's default on_boot hook, run on every agent boot; // - gc-list-on-boot-workflow: the same recovery, workflow lane; // - mc-list-title: the PR-intake formula's `--title "$SUBJECT"`; +// - gc-polecat-title-contains: the polecat pre-flight's open-bead lookup, +// `--title-contains "$SYMPTOM_KEY"`, folded onto `title`; +// - title-and-title-contains-equal (not a corpus shape): one value under +// both title flags is one `title`; // - mc-list-id: the pr-review skill's `--id `. var listScopeShapes = []struct { corpusID string @@ -31,6 +35,15 @@ var listScopeShapes = []struct { {"mc-list-title", []string{"list", "--all", "--include-infra", "--json", "--limit", "10", "--title", "CHILD-A"}, "issues.list.title", []string{"child-a"}}, + // --title-contains folds onto the same `title` parameter: case-insensitive + // substring, and the status set leaves the closed child-c out. + {"gc-polecat-title-contains", + []string{"list", "--title-contains", "CHILD", "--status", "open,in_progress,blocked", "--json"}, + "issues.list.title", []string{"child-a", "child-b", "child-x"}}, + // The same value under both flags is one predicate. + {"title-and-title-contains-equal", + []string{"list", "--all", "--title", "child-a", "--title-contains", "child-a", "--json"}, + "issues.list.title", []string{"child-a"}}, // A closed id in the set is listed under --all; ids outside it are not. {"mc-list-id", []string{"list", "--all", "--id", "{child-c},{loner}", "--json", "--limit", "0"}, @@ -88,6 +101,19 @@ func TestE2E_ListScopeFiltersOverHTTP(t *testing.T) { }) } + // The listing publishes one title parameter: a differing pair refuses + // rather than drop either half. + t.Run("differing title pair refuses", func(t *testing.T) { + args := []string{"list", "--all", "--title", "child-a", "--title-contains", "child-b", "--json"} + r := remote.run(nil, args...) + if r.code == 0 { + t.Fatalf("http: bd %s exited 0; one half of the pair was dropped: %s", strings.Join(args, " "), r.stdout) + } + if !strings.Contains(r.stdout+r.stderr, "TitleContains") { + t.Errorf("the refusal does not name TitleContains: stdout=%s stderr=%s", r.stdout, r.stderr) + } + }) + t.Run("MC server mask refuses before dialing", func(t *testing.T) { proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) f := newHTTPFixture(t, bin, proxyAddr) diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 29fcfa1f4..269d45eda 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -183,7 +183,7 @@ The listing publishes far more filters than the v0 wire does; every one the wire | E-ListRequest.SpecPrefix | refuse | `ListRequest.SpecPrefix` | a spec-id prefix restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.LabelPattern | refuse | `ListRequest.LabelPattern` | a glob label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.LabelRegex | refuse | `ListRequest.LabelRegex` | a regex label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — as for LabelPattern, listReadyWork publishes label_regex and the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.TitleContains | refuse | `ListRequest.TitleContains` | a title substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.TitleContains | refuse | `ListRequest.TitleContains` | a title substring that differs from the title filter refuses | listIssues publishes one title parameter. A TitleContains alone, or equal to TitleSearch once both are lowered, is folded onto `title` before encoding (FoldTitleContains: both compile to the same LOWER(title) LIKE predicate on the strings.ToLower of the value), so only a pair that still differs reaches this row, and sending either half alone would answer a wider listing | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.DescContains | refuse | `ListRequest.DescContains` | a description substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.NotesContains | refuse | `ListRequest.NotesContains` | a notes substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.ExternalContains | refuse | `ListRequest.ExternalContains` | an external-ref substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel index f09bdd196..d0de56d70 100644 --- a/internal/httpclient/encode/BUILD.bazel +++ b/internal/httpclient/encode/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "ledger.go", "ledger_fields.go", "ledger_render.go", + "list_fold.go", "parentwalk.go", "skew.go", "table.go", @@ -25,6 +26,7 @@ go_test( srcs = [ "bijection_test.go", "ledger_render_test.go", + "list_fold_test.go", "oracle_test.go", "parentwalk_test.go", "roundtrip_test.go", diff --git a/internal/httpclient/encode/ledger_fields.go b/internal/httpclient/encode/ledger_fields.go index 18709c6fc..642d5afcb 100644 --- a/internal/httpclient/encode/ledger_fields.go +++ b/internal/httpclient/encode/ledger_fields.go @@ -113,7 +113,8 @@ func listRequestRows() []Row { noWireList+" — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through"), listRefusal("LabelRegex", "a regex label filter refuses", noWireList+" — as for LabelPattern, listReadyWork publishes label_regex and the listing does not"), - listRefusal("TitleContains", "a title substring filter refuses", noWireList), + listRefusal("TitleContains", "a title substring that differs from the title filter refuses", + "listIssues publishes one title parameter. A TitleContains alone, or equal to TitleSearch once both are lowered, is folded onto `title` before encoding (FoldTitleContains: both compile to the same LOWER(title) LIKE predicate on the strings.ToLower of the value), so only a pair that still differs reaches this row, and sending either half alone would answer a wider listing"), listRefusal("DescContains", "a description substring filter refuses", noWireList), listRefusal("NotesContains", "a notes substring filter refuses", noWireList), listRefusal("ExternalContains", "an external-ref substring filter refuses", noWireList), diff --git a/internal/httpclient/encode/list_fold.go b/internal/httpclient/encode/list_fold.go new file mode 100644 index 000000000..f64855991 --- /dev/null +++ b/internal/httpclient/encode/list_fold.go @@ -0,0 +1,41 @@ +package encode + +import ( + "strings" + + "github.com/steveyegge/beads/issueops" +) + +// FoldTitleContains rewrites ListRequest.TitleContains onto TitleSearch when +// the pair compiles to one clause, so `bd list --title-contains X` rides the +// listing's existing `title` parameter (behind issues.list.title) instead of +// refusing. +// +// The fold is exact, not an approximation: the store compiles both members to +// the identical predicate, `LOWER(title) LIKE '%' || lower(v) || '%'` +// (internal/storage/sqlbuild, which lowers v with strings.ToLower), and +// workapi.BuildListFilter copies each straight through. So when TitleSearch is +// empty, or equal to TitleContains once both are lowered the same way, the +// request without TitleContains selects exactly the same rows. The comparison +// is strings.ToLower equality rather than strings.EqualFold because only the +// former is the store's own lowering: EqualFold also matches pairs such as +// "ſ" and "s", which lower to different patterns. +// +// When both are set and differ the request is returned unchanged: the listing +// publishes one title parameter, and the encoder refuses the leftover +// TitleContains (E-ListRequest.TitleContains) rather than drop either half. +// +// The request is taken and returned by value; the caller's copy is never +// mutated. +func FoldTitleContains(req issueops.ListRequest) issueops.ListRequest { + if req.TitleContains == "" { + return req + } + switch { + case req.TitleSearch == "": + req.TitleSearch, req.TitleContains = req.TitleContains, "" + case strings.ToLower(req.TitleSearch) == strings.ToLower(req.TitleContains): + req.TitleContains = "" + } + return req +} diff --git a/internal/httpclient/encode/list_fold_test.go b/internal/httpclient/encode/list_fold_test.go new file mode 100644 index 000000000..aee159feb --- /dev/null +++ b/internal/httpclient/encode/list_fold_test.go @@ -0,0 +1,59 @@ +package encode + +import ( + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// `bd list --title-contains X` folds onto the listing's `title` parameter: the +// store compiles TitleContains and TitleSearch to one predicate, so the fold +// is exact. A pair that differs other than in letter case still refuses, +// because the listing publishes one title parameter and dropping either half +// would widen the answer. +func TestFoldTitleContainsRidesTheTitleParameter(t *testing.T) { + for _, tc := range []struct { + name string + req issueops.ListRequest + wantTitle []string + refused bool + }{ + {name: "title-contains alone", req: issueops.ListRequest{TitleContains: "Routed"}, wantTitle: []string{"Routed"}}, + {name: "equal pair sends one title", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "Routed"}, wantTitle: []string{"Routed"}}, + // The store lowers both values with strings.ToLower, so a pair that + // differs only in letter case is one predicate too. + {name: "pair differing only in case sends one title", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "rOUTED"}, wantTitle: []string{"Routed"}}, + {name: "title alone is untouched", req: issueops.ListRequest{TitleSearch: "Routed"}, wantTitle: []string{"Routed"}}, + {name: "neither sends no title", req: issueops.ListRequest{}}, + {name: "differing pair refuses", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "task"}, refused: true}, + // strings.EqualFold matches "ſ" (long s) with "s", but the store lowers + // them to different patterns, so this pair is two predicates. + {name: "pair equal only under case folding refuses", req: issueops.ListRequest{TitleSearch: "ſtep", TitleContains: "step"}, refused: true}, + } { + t.Run(tc.name, func(t *testing.T) { + folded := FoldTitleContains(tc.req) + params, err := ListParams(folded) + if tc.refused { + var refused *RefusedError + if !errors.As(err, &refused) { + t.Fatalf("ListParams(%+v) error = %v, want a *RefusedError", folded, err) + } + if refused.Row.ID != "E-ListRequest.TitleContains" { + t.Fatalf("refused by row %q, want E-ListRequest.TitleContains", refused.Row.ID) + } + return + } + if err != nil { + t.Fatalf("ListParams(%+v): %v", folded, err) + } + if got := params["title"]; !reflect.DeepEqual(got, tc.wantTitle) { + t.Errorf("title = %q, want %q", got, tc.wantTitle) + } + if got, ok := params["title_contains"]; ok { + t.Errorf("listIssues carries no title_contains parameter, but the encoder sent %q", got) + } + }) + } +} diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index b24c44365..a18ad5907 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -345,6 +345,10 @@ func listTable() Table { param("ExcludeLabels", "exclude_label"), refused("LabelPattern", "E-ListRequest.LabelPattern"), refused("LabelRegex", "E-ListRequest.LabelRegex"), + // A TitleContains alone, or equal to TitleSearch up to letter + // case, never reaches the encoder: httpReader.List folds it onto + // `title` first (FoldTitleContains). What is left is a pair that + // differs. refused("TitleContains", "E-ListRequest.TitleContains"), refused("DescContains", "E-ListRequest.DescContains"), refused("NotesContains", "E-ListRequest.NotesContains"), diff --git a/internal/httpclient/list_scope_test.go b/internal/httpclient/list_scope_test.go index c4f67ce07..87e775f40 100644 --- a/internal/httpclient/list_scope_test.go +++ b/internal/httpclient/list_scope_test.go @@ -25,6 +25,10 @@ var listScopeCases = []struct { func(r *issueops.ListRequest) { r.NoAssignee = true }, []string{"true"}}, {"TitleSearch", "title", wire.CapListTitle, func(r *issueops.ListRequest) { r.TitleSearch = "Routed open task" }, []string{"Routed open task"}}, + // `--title-contains` compiles to the same predicate as `--title`, so it + // folds onto `title` behind the same token. + {"TitleContains", "title", wire.CapListTitle, + func(r *issueops.ListRequest) { r.TitleContains = "Routed" }, []string{"Routed"}}, {"IDFilter", "id", wire.CapListIDs, // Split the way the role splits it, one `id` value per member. func(r *issueops.ListRequest) { r.IDFilter = "e2e-r1, e2e-a1,e2e-r1" }, []string{"e2e-r1", "e2e-a1"}}, @@ -98,3 +102,23 @@ func TestListScopeFiltersRefuseLocallyWhenTheServerLacksTheToken(t *testing.T) { }) } } + +// TestListTitleContainsThatDiffersFromTitleRefusesBeforeDialing: the listing +// publishes one title parameter, so a `--title` and `--title-contains` that +// differ cannot both ride it. The request refuses, naming the member, without +// a round trip and without dropping either half. +func TestListTitleContainsThatDiffersFromTitleRefusesBeforeDialing(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListTitle}} + w := newCountingWire(`{"items":[],"has_more":false}`) + req := issueops.ListRequest{TitleSearch: "Routed", TitleContains: "task"} + _, err := listRoleWithSnapshot(t, w, served).List(t.Context(), req) + if err == nil { + t.Fatal("List with a differing --title/--title-contains pair succeeded; one half was dropped") + } + if !strings.Contains(err.Error(), "TitleContains") { + t.Errorf("refusal %q does not name TitleContains", err) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0", len(w.dialed)) + } +} diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index 9c38282f7..b66bef781 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -63,7 +63,12 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue } // List serves listIssues, through the pager in list_walk.go. +// +// `--title-contains` is folded onto `title` first (encode.FoldTitleContains: +// the two compile to one predicate), so it is gated by issues.list.title like +// `--title` is, and refused before the dial against a server without it. func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + req = encode.FoldTitleContains(req) if err := r.refuseUnservedListFilters(ctx, req); err != nil { return issueops.IssuePage{}, err } @@ -77,8 +82,10 @@ func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueop // tokenedListFilters are the ListRequest members listIssues carries only on a // server advertising a behavior token of their own, each with that token: // `bd search`'s text (S26) and `bd list --title`, `--id` and `--no-assignee` -// (S15b). Order decides which token a request carrying several names; each -// refusal is complete on its own, so any order is honest. +// (S15b). `--title-contains` has no entry: List folds it onto TitleSearch +// first (encode.FoldTitleContains), so it is gated by issues.list.title. +// Order decides which token a request carrying several names; each refusal is +// complete on its own, so any order is honest. var tokenedListFilters = []struct { token string set func(issueops.ListRequest) bool diff --git a/internal/httpclient/served_list_test.go b/internal/httpclient/served_list_test.go index 751bee628..f364de85f 100644 --- a/internal/httpclient/served_list_test.go +++ b/internal/httpclient/served_list_test.go @@ -1478,7 +1478,9 @@ func TestServedReaderRefusesTheFiltersTheWireCannotCarry(t *testing.T) { {"PinnedFlag", issueops.ListRequest{PinnedFlag: true}}, {"PriorityMax", issueops.ListRequest{PriorityMax: &priority}}, {"UpdatedAfter", issueops.ListRequest{UpdatedAfter: &cutoff}}, - {"TitleContains", issueops.ListRequest{TitleContains: "anything"}}, + // A TitleContains alone folds onto `title`; only a pair that differs + // is left with no parameter to ride. + {"TitleContains", issueops.ListRequest{TitleSearch: "something", TitleContains: "anything"}}, {"ExcludeTypes", issueops.ListRequest{ExcludeTypes: []string{"chore"}}}, } { page, err := reader.List(ctx, test.req) @@ -1497,6 +1499,43 @@ func TestServedReaderRefusesTheFiltersTheWireCannotCarry(t *testing.T) { } } +// TestServedListTitleContainsAnswersWhatTitleAnswers: `bd list +// --title-contains X` folds onto listIssues `title`, and the store compiles +// both to the same case-insensitive substring predicate, so over the wire the +// two flags answer the same rows, as do both flags set to one value, in the +// same or a different letter case. +func TestServedListTitleContainsAnswersWhatTitleAnswers(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + const scope = "title-contains" + for _, id := range []string{listID("tcmatch", "a"), listID("tcmatch", "b"), listID("other", "c")} { + seedListIssue(t, ctx, c, id, scope, time.Time{}, 2) + } + limit := 0 + answer := func(req issueops.ListRequest) []string { + t.Helper() + req.Labels, req.Limit = []string{scope}, &limit + page, err := reader.List(ctx, req) + if err != nil { + t.Fatalf("List(%+v): %v", req, err) + } + ids := pageIDs(page) + slices.Sort(ids) + return ids + } + want := []string{listID("tcmatch", "a"), listID("tcmatch", "b")} + for name, req := range map[string]issueops.ListRequest{ + "title": {TitleSearch: "LSTCMATCH"}, + "title-contains": {TitleContains: "LSTCMATCH"}, + "both, the same value": {TitleSearch: "lstcmatch", TitleContains: "lstcmatch"}, + "both, differing only in case": {TitleSearch: "LSTCMATCH", TitleContains: "lstcmatch"}, + } { + if got := answer(req); !slices.Equal(got, want) { + t.Errorf("%s: ids %v, want %v", name, got, want) + } + } +} + // seedListWisp seeds one EPHEMERAL row through the reference store, carrying // the same label scope its durable siblings do. The flag is set here rather // than by the caller for servedEnv.createWisp's reason: the plane is the seed From b2d5cff6c00b5a8434cf7d11128d17def4fcfa51 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 17 Oct 2026 10:08:48 +0000 Subject: [PATCH 07/54] fix(gate): add-waiter retries contention the store gave up replaying (#7648) * fix(gate): add-waiter retries contention the store gave up replaying bd gate add-waiter writes the whole waiter list guarded by the version it read and retried only ErrVersionMismatch. A convoy adder therefore failed when its write ended in contention the backend stopped replaying: a serialization failure or deadlock (Dolt/MySQL 1213/1205, Postgres 40001/40P01, uow.IsSerializationError) on a direct route, or a 503 busy (wire.ErrBusy) over HTTP, which bd serve sends when its admission limiter refuses or its transaction retry budget is spent. gateAddWaiterLostRound now retries those like a version miss, under the same jittered backoff and 20-write bound, and a busy answer's Retry-After raises that round's wait, capped at 2s (gateAddWaiterRetryWait). The retry is safe because of how a round is built, not because a 503 means nothing was written: the client maps any 503 to wire.ErrBusy, and a gateway can answer 503 after the write committed. Each round re-reads the gate, deduplicates and writes the whole list guarded by the version it read, so a landed write is found on the re-read (the waiter is never added twice) and a stale list never overwrites a newer one. A server fault (500, e.g. an indeterminate commit) and an unreachable database are still returned at once. TestAddGateWaiterRetriesTransientContention pins the three retried spellings, the bound, busy-after-commit (one write, the waiter once), the 500 / db-unavailable / other-SQLSTATE non-retries, and the capped Retry-After. It is red without the fix. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(gate): bound add-waiter's contention retries and honor Retry-After Review fixups for #7648. - A write lost to contention reaches addGateWaiter only after the store or bd serve has spent a whole replay budget on it (withRetryTx: 5s, 15s in server mode; bd serve's: 15s), so retrying it under the 20-write bound a version miss gets could hold an adder for minutes (about five on the direct server-mode route). Contended writes now get their own bound, gateAddWaiterContentionRetries = 2, like the HTTP client's for a busy read. They still count against gateAddWaiterMaxAttempts, and version misses keep the 20. The const doc states the worst case: three 15s budgets and two 5s waits, about a minute. - A busy answer's Retry-After is waited out as sent, plus the jittered backoff, instead of being capped at 2s. bd serve asks for 5s after a spent budget because a convoy back sooner holds slots and starves reads, and the wire client already bounds the header. The jitter keeps adders answered busy together from retrying together. - When the last write the bounds allow is answered busy, one more read checks the gate before the command fails, with no wait first. A 503 (a gateway's, say) can answer a write that committed, and every earlier round's re-read would have found it; the last write has no round left. A waiter found there is reported as already registered, and a read that fails confirms nothing. - The exhaustion error says "changed on every one of N attempts" only when no write was contended. Otherwise it reads "no write to X was confirmed in N attempts (K answered busy or lost to write contention)": it names the contention, so the operator of a busy server is not sent looking for a concurrent writer, and it does not claim that nothing landed. - gateAddWaiterContended names what maps to each class: Dolt/MySQL 1213/1205 and SQLSTATE 40001/40P01 on the direct route, and wire.ErrBusy from bd serve's busy answer or from a 503 with no code or an unknown one (db_unavailable and events_watch_saturated map to their own sentinels). It also says why Dolt's 1105 "@autocommit transaction rolled back" is not admitted: Dolt raises it for conflicts that a merge inside the transaction left behind, while a write lost to a concurrent writer comes back as 1213. - Tests: 1213 and 1205 rows built as *mysql.MySQLError (the default Dolt route's shape), the contention bound for busy and for 1213 with no wait after the last contended write, version misses not spending it, the three exhaustion wordings, a 5s Retry-After waited whole plus the jitter, a busy answer to the last write at either bound that landed and is found by the extra read, and a failed extra read that confirms nothing. Thirteen mutants each fail at least one subtest: no bound, busy only counted, version misses counted, no jitter, the 2s cap, the old wording, an off-by-one bound, no extra read, an extra read that ignores the waiter, one that takes a failed read as landed, a wait before it, a wait at the end of each round, and the bound checked after the wait. - CHANGELOG entry updated to match. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- CHANGELOG.md | 17 ++ cmd/bd/BUILD.bazel | 3 + cmd/bd/gate.go | 123 +++++++++- cmd/bd/gate_add_waiter_contention_test.go | 278 ++++++++++++++++++++++ 4 files changed, 411 insertions(+), 10 deletions(-) create mode 100644 cmd/bd/gate_add_waiter_contention_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f1809fd9..f6b79d897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -616,6 +616,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd gate add-waiter` survives contention the store gave up replaying.** + It retried only a version miss, so a convoy adder failed when its guarded + write ended in a serialization failure or deadlock the backend stopped + replaying (Dolt/MySQL 1213/1205, Postgres 40001/40P01), or, over HTTP, in a + 503 busy. Those are now retried, but at most twice: each arrives only after + the store or `bd serve` has spent a whole replay budget on the write (up to + 15s), so the 20-write bound a version miss gets could hold an adder for + minutes. Each retry waits the jittered backoff plus a busy answer's + Retry-After, as `bd serve` sent it. The retry is safe because every round + re-reads the gate, deduplicates and writes the whole list guarded by the + version it read: a 503 is not proof that nothing was written (a gateway can + answer one after the commit), and a write that did land is found on the + re-read, so the waiter is never added twice; when the last write allowed is + answered busy, one more read checks for it before the command fails. A + server fault (500) and an unreachable database are still returned at once, + and once the retries run out the error says no write was confirmed and + names the write contention, instead of saying the gate kept changing. - `bd list --title-contains X` works against an HTTP workspace. The client folds it onto the listing's existing `title` parameter (the store compiles both flags to the same case-insensitive substring predicate), so it needs diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 164b7c316..9b2af31f8 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -416,6 +416,7 @@ go_library( "//internal/hooks", "//internal/httpapi", "//internal/httpclient", + "//internal/httpclient/wire", "//internal/jira", "//internal/linear", "//internal/lockfile", @@ -801,6 +802,7 @@ go_test( "formula_help_test.go", "formula_primitives_test.go", "formula_schema_test.go", + "gate_add_waiter_contention_test.go", "gate_check_listing_test.go", "gate_check_proxied_integration_test.go", "gate_discover_test.go", @@ -1348,6 +1350,7 @@ go_test( "//internal/httpapi", "//internal/httpclient", "//internal/httpclient/encode", + "//internal/httpclient/wire", "//internal/linear", "//internal/metrics", "//internal/migration", diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 25fe25e0a..959c6489e 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -15,9 +15,11 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/httpclient/wire" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/issueops" @@ -294,14 +296,43 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio // seconds per read and write), and a wall-clock budget would fail a convoy on // a slow server after only two or three rounds. Each round has a winner, so // the bound carries at least that many simultaneous adders; the jitter -// spreads them further. The waits themselves add up to at most about 6s, and -// a gate rewritten without pause still fails after the last attempt. +// spreads them further. The backoffs themselves add up to at most about 6s, +// and a gate rewritten without pause still fails after the last attempt. +// +// A write lost to contention the store gave up replaying (see +// gateAddWaiterContended) is not one quick round, so it gets a much tighter +// bound of its own, gateAddWaiterContentionRetries. It reaches this loop only +// after the store or bd serve has spent a whole replay budget on it (up to +// 15s, the server-mode Dolt store's and bd serve's), and the wait after a +// busy answer includes the server's Retry-After (5s once bd serve's budget is +// spent). Two retries, like the HTTP client's for a busy read, hold that tail +// to three such writes (three 15s budgets and two 5s waits: about a minute), +// where twenty would take over six minutes. They count against +// gateAddWaiterMaxAttempts too. const ( - gateAddWaiterMaxAttempts = 20 - gateAddWaiterBackoffBase = 10 * time.Millisecond - gateAddWaiterBackoffCap = 400 * time.Millisecond + gateAddWaiterMaxAttempts = 20 + gateAddWaiterContentionRetries = 2 + gateAddWaiterBackoffBase = 10 * time.Millisecond + gateAddWaiterBackoffCap = 400 * time.Millisecond ) +// gateAddWaiterRetryWait is the wait before retry number retry after the +// failed write err: the jittered backoff, plus the server's Retry-After when +// err is a busy answer that carries one. bd serve sends Retry-After on its +// busy answers: 1s when its admission limiter is saturated, and 5s when its +// transaction retry budget is spent, because a convoy back sooner would hold +// slots while it waited, starving reads. It is honored as sent: the wire +// client already bounds the header (wire.Options.MaxRetryAfter). The jitter on +// top keeps adders that were answered busy together from retrying together. +func gateAddWaiterRetryWait(retry int, err error) time.Duration { + wait := gateAddWaiterBackoff(retry) + var problem *wire.ProblemError + if errors.Is(err, wire.ErrBusy) && errors.As(err, &problem) { + wait += problem.RetryAfter + } + return wait +} + // gateAddWaiterBackoff is the wait before retry number retry (1-based): // jittered in [d/2, d) around d = base * 2^(retry-1), capped. func gateAddWaiterBackoff(retry int) time.Duration { @@ -342,10 +373,10 @@ var gateAddWaiterSleep = func(ctx context.Context, d time.Duration) error { // the role stores the list it is given. func addGateWaiter(ctx context.Context, reader issueops.Reader, lifecycle issueops.Lifecycle, gateID, waiter string) (bool, error) { var lastErr error - attempts := 0 - for attempts < gateAddWaiterMaxAttempts { + attempts, contended := 0, 0 + for attempts < gateAddWaiterMaxAttempts && contended <= gateAddWaiterContentionRetries { if attempts > 0 { - if err := gateAddWaiterSleep(ctx, gateAddWaiterBackoff(attempts)); err != nil { + if err := gateAddWaiterSleep(ctx, gateAddWaiterRetryWait(attempts, lastErr)); err != nil { return false, fmt.Errorf("updating gate: %w", err) } } @@ -376,12 +407,84 @@ func addGateWaiter(ctx context.Context, reader issueops.Reader, lifecycle issueo if err == nil { return true, nil } - if !errors.Is(err, issueops.ErrVersionMismatch) { + if !gateAddWaiterLostRound(err) { return false, fmt.Errorf("updating gate: %w", err) } + if gateAddWaiterContended(err) { + contended++ + } lastErr = err } - return false, fmt.Errorf("updating gate: %s changed on every one of %d attempts: %w", gateID, attempts, lastErr) + if errors.Is(lastErr, wire.ErrBusy) && gateAddWaiterLanded(ctx, reader, gateID, waiter) { + return false, nil + } + return false, gateAddWaiterExhausted(gateID, attempts, contended, lastErr) +} + +// gateAddWaiterLanded reports whether waiter is on the gate gateID, read once +// after addGateWaiter's last write was answered busy. Such a write may have +// landed (see gateAddWaiterLostRound), and every earlier round's re-read +// would have found it, but the last write has no round left; so it gets this +// one read, with no wait first. A failed read reports false. +func gateAddWaiterLanded(ctx context.Context, reader issueops.Reader, gateID, waiter string) bool { + details, err := reader.Get(ctx, issueops.GetRequest{ID: gateID}) + return err == nil && slices.Contains(details.Issue.Waiters, waiter) +} + +// gateAddWaiterLostRound reports whether a failed guarded write is a lost +// round the loop retries: a version miss, or a write lost to contention the +// store gave up replaying (gateAddWaiterContended). +// +// Retrying is safe because of how a round is built, NOT because every busy +// answer means nothing was written. bd serve's own busy answers are no-write, +// but a gateway in front of the server can answer 503 after the write +// committed, and the client maps that uncoded 503 to wire.ErrBusy too. Each +// round re-reads the gate, deduplicates, and writes the whole list guarded by +// the row version it read, so a write that did land is found on the re-read +// (the waiter is already there: nothing more is written) and a stale list can +// never overwrite a newer one. So a retry never adds a waiter twice and never +// drops another adder's. +// +// Everything else is returned at once, including the classes a retry would +// also survive but that are not contention: a refusal, a database bd serve +// cannot reach (wire.ErrDBUnavailable), and a server fault such as an +// indeterminate commit (wire.ErrServerFault, a 500). Those are for the caller +// to see, not for a convoy loop to hammer. +func gateAddWaiterLostRound(err error) bool { + return errors.Is(err, issueops.ErrVersionMismatch) || gateAddWaiterContended(err) +} + +// gateAddWaiterContended reports whether a failed guarded write was lost to +// contention the store gave up replaying. On the direct route that is a +// serialization failure or deadlock, which guarantees a rollback (Dolt and +// MySQL report 1213/1205, a SERIALIZABLE store such as Postgres SQLSTATE +// 40001/40P01): the store's transaction envelope replays one until its budget +// is spent and then returns it. Over HTTP it is wire.ErrBusy, which the client +// maps from bd serve's busy answer (a refused admission, or a spent retry +// budget) and from a 503 whose code it does not know or that has none, such as +// a gateway's. A 503 coded db_unavailable or events_watch_saturated maps to its +// own sentinel, not to this one. +// +// Dolt's 1105 "@autocommit transaction rolled back", which the direct route's +// store also replays, is deliberately not admitted: Dolt raises it for +// conflicts that a merge inside the transaction left behind, while a write +// lost to a concurrent writer, the convoy's case, comes back as 1213. +func gateAddWaiterContended(err error) bool { + return uow.IsSerializationError(err) || errors.Is(err, wire.ErrBusy) +} + +// gateAddWaiterExhausted is addGateWaiter's error once it stops retrying: +// after attempts writes, contended of them answered busy or lost to +// contention, the last with lastErr. Only when none was contended did the gate +// change under every write, so only then does the error say that; otherwise it +// names the contention, so that the operator of a busy server is not sent +// looking for a concurrent writer. It says no write was confirmed, not that +// none landed, because a busy answer does not prove that nothing was written. +func gateAddWaiterExhausted(gateID string, attempts, contended int, lastErr error) error { + if contended == 0 { + return fmt.Errorf("updating gate: %s changed on every one of %d attempts: %w", gateID, attempts, lastErr) + } + return fmt.Errorf("updating gate: no write to %s was confirmed in %d attempts (%d answered busy or lost to write contention): %w", gateID, attempts, contended, lastErr) } // renderGateWaiterAlready and renderGateWaiterAdded are shared by the direct diff --git a/cmd/bd/gate_add_waiter_contention_test.go b/cmd/bd/gate_add_waiter_contention_test.go new file mode 100644 index 000000000..0113b46c8 --- /dev/null +++ b/cmd/bd/gate_add_waiter_contention_test.go @@ -0,0 +1,278 @@ +package main + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "testing" + "time" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// sqlStateError is a driver error carrying a SQLSTATE, the shape pgconn.PgError +// and every other driver error uow.IsSerializationError reads. +type sqlStateError struct{ state string } + +func (e *sqlStateError) Error() string { + return "ERROR: could not serialize access (SQLSTATE " + e.state + ")" +} +func (e *sqlStateError) SQLState() string { return e.state } + +// contendedGateRoles fails the first len(errs) updates with errs, then +// behaves as fakeGateRoles. A failed update writes nothing, except failed +// update number landed (1-based): it lands first and is answered with its +// error anyway, as a gateway's 503 can answer a write the server committed. +// Once every one of errs is spent, a read answers readErr if it is set. +type contendedGateRoles struct { + *fakeGateRoles + errs []error + landed int + failed int + readErr error +} + +func (c *contendedGateRoles) Get(ctx context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + if c.readErr != nil && len(c.errs) == 0 { + return nil, c.readErr + } + return c.fakeGateRoles.Get(ctx, req) +} + +func (c *contendedGateRoles) Update(ctx context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + if len(c.errs) > 0 { + err := c.errs[0] + c.errs = c.errs[1:] + c.failed++ + if c.failed == c.landed { + if _, applyErr := c.fakeGateRoles.Update(ctx, req); applyErr != nil { + return issueops.UpdateResult{}, applyErr + } + return issueops.UpdateResult{}, err + } + c.updates = append(c.updates, req) + return issueops.UpdateResult{}, err + } + return c.fakeGateRoles.Update(ctx, req) +} + +// TestAddGateWaiterRetriesTransientContention: contention the store gave up +// replaying is one more lost round, not a failure, but a costly one. The +// store's transaction envelope replays a serialization failure or deadlock +// (Dolt and MySQL 1213/1205, SQLSTATE 40001/40P01) until its budget is spent +// and then returns it; over HTTP bd serve answers that spent budget, or a +// refused admission, as 503 busy (wire.ErrBusy). The loop retries them under +// the same backoff as a version miss, but only gateAddWaiterContentionRetries +// times, because each such write has already spent a whole replay budget. +// What makes a retry safe is the round itself (re-read, deduplicate, +// version-guarded whole-list write), which the busy-after-commit case pins: a +// 503 is not proof that nothing was written. +func TestAddGateWaiterRetriesTransientContention(t *testing.T) { + setGateTestActor(t) + origSleep := gateAddWaiterSleep + t.Cleanup(func() { gateAddWaiterSleep = origSleep }) + var sleeps []time.Duration + gateAddWaiterSleep = func(_ context.Context, d time.Duration) error { + sleeps = append(sleeps, d) + return nil + } + gate := func() *types.Issue { + return &types.Issue{ID: "bd-g", IssueType: "gate", Waiters: []string{"a"}, RowVersion: 7} + } + doltSerialization := fmt.Errorf("update: %w", &mysql.MySQLError{Number: 1213, Message: "serialization failure: this transaction conflicts with a committed transaction from another client, try restarting transaction"}) + + for _, tt := range []struct { + name string + err error + }{ + {"serialization failure", fmt.Errorf("update: %w", &sqlStateError{state: "40001"})}, + {"deadlock", fmt.Errorf("update: %w", &sqlStateError{state: "40P01"})}, + {"Dolt serialization failure (1213)", doltSerialization}, + {"lock wait timeout (1205)", fmt.Errorf("update: %w", &mysql.MySQLError{Number: 1205, Message: "Lock wait timeout exceeded; try restarting transaction"})}, + {"bd serve busy", fmt.Errorf("updateIssue: %w", wire.ErrBusy)}, + } { + t.Run(tt.name, func(t *testing.T) { + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{tt.err, tt.err}} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added after two contended rounds", added, err) + } + if len(roles.updates) != 3 || len(sleeps) != 2 { + t.Fatalf("updates = %d, sleeps = %d; want 3 writes with a backoff before each retry", len(roles.updates), len(sleeps)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b]", got) + } + }) + } + + for _, tt := range []struct { + name string + err error + }{ + {"bd serve busy", wire.ErrBusy}, + {"Dolt serialization failure", doltSerialization}, + } { + t.Run(tt.name+" gets only the contention retries", func(t *testing.T) { + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: slices.Repeat([]error{tt.err}, gateAddWaiterContentionRetries+2)} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, tt.err) { + t.Fatalf("addGateWaiter error = %v, want the last contended write's %v", err, tt.err) + } + if want := gateAddWaiterContentionRetries + 1; len(roles.updates) != want { + t.Fatalf("updates = %d, want %d: each contended write already spent a whole replay budget", len(roles.updates), want) + } + if len(sleeps) != gateAddWaiterContentionRetries { + t.Errorf("sleeps = %v, want one per retry and none after the last contended write", sleeps) + } + }) + } + + t.Run("version misses do not spend the contention retries", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{ + wire.ErrBusy, wire.ErrBusy, issueops.ErrVersionMismatch, issueops.ErrVersionMismatch, + }} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added || len(roles.updates) != 5 { + t.Fatalf("addGateWaiter = %v, %v after %d updates; want added on the fifth write", added, err, len(roles.updates)) + } + }) + + for _, tt := range []struct { + name string + errs []error + want string + is error + }{ + { + "a gate that changed under every write", + slices.Repeat([]error{issueops.ErrVersionMismatch}, gateAddWaiterMaxAttempts), + "updating gate: bd-g changed on every one of 20 attempts: ", + issueops.ErrVersionMismatch, + }, + { + "writes lost to contention", + slices.Repeat([]error{wire.ErrBusy}, gateAddWaiterContentionRetries+1), + "updating gate: no write to bd-g was confirmed in 3 attempts (3 answered busy or lost to write contention): ", + wire.ErrBusy, + }, + { + "one contended write among version misses", + append([]error{wire.ErrBusy}, slices.Repeat([]error{issueops.ErrVersionMismatch}, gateAddWaiterMaxAttempts-1)...), + "updating gate: no write to bd-g was confirmed in 20 attempts (1 answered busy or lost to write contention): ", + issueops.ErrVersionMismatch, + }, + } { + t.Run("the exhaustion error after "+tt.name, func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: tt.errs} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, tt.is) { + t.Fatalf("addGateWaiter error = %v, want it to wrap %v", err, tt.is) + } + if got := err.Error(); got != tt.want+tt.is.Error() { + t.Errorf("addGateWaiter error = %q, want %q", got, tt.want+tt.is.Error()) + } + }) + } + + t.Run("a busy answer to a write that landed adds the waiter once", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{wire.ErrBusy}, landed: 1} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil { + t.Fatalf("addGateWaiter error = %v, want the retry's re-read to find the landed waiter", err) + } + if added { + t.Errorf("addGateWaiter reported a second add; the re-read must find b already registered") + } + if len(roles.updates) != 1 { + t.Errorf("updates = %d, want only the write that landed", len(roles.updates)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b] with b exactly once", got) + } + }) + + for _, tt := range []struct { + name string + errs []error + }{ + {"the contention bound", slices.Repeat([]error{wire.ErrBusy}, gateAddWaiterContentionRetries+1)}, + {"the attempt bound", append(slices.Repeat([]error{issueops.ErrVersionMismatch}, gateAddWaiterMaxAttempts-1), wire.ErrBusy)}, + } { + t.Run("a busy answer to the last write, at "+tt.name+", that landed is found by one more read", func(t *testing.T) { + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: tt.errs, landed: len(tt.errs)} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || added { + t.Fatalf("addGateWaiter = %v, %v; want no error and no second add once the last read finds b", added, err) + } + if len(roles.updates) != len(tt.errs) || len(sleeps) != len(tt.errs)-1 { + t.Fatalf("updates = %d, sleeps = %d; want %d writes and no wait before the last read", len(roles.updates), len(sleeps), len(tt.errs)) + } + if got := roles.issues["bd-g"].Waiters; !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("waiters = %v, want [a b] with b exactly once", got) + } + }) + } + + t.Run("a failed read after the last busy write confirms nothing", func(t *testing.T) { + errs := slices.Repeat([]error{wire.ErrBusy}, gateAddWaiterContentionRetries+1) + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: errs, landed: len(errs), readErr: fmt.Errorf("getIssue: %w", wire.ErrBusy)} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + want := "updating gate: no write to bd-g was confirmed in 3 attempts (3 answered busy or lost to write contention): " + wire.ErrBusy.Error() + if err == nil || err.Error() != want { + t.Fatalf("addGateWaiter error = %v, want %q", err, want) + } + }) + + for _, tt := range []struct { + name string + err error + }{ + {"a server fault (500, an indeterminate commit)", fmt.Errorf("updateIssue: %w", wire.ErrServerFault)}, + {"an unreachable database", fmt.Errorf("updateIssue: %w", wire.ErrDBUnavailable)}, + } { + t.Run(tt.name+" is not retried", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{tt.err}} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if !errors.Is(err, tt.err) || len(roles.updates) != 1 { + t.Fatalf("addGateWaiter = %v after %d updates, want %v after one write", err, len(roles.updates), tt.err) + } + }) + } + + t.Run("a busy answer's Retry-After is waited out whole, plus the jitter", func(t *testing.T) { + busy := &wire.ProblemError{Status: 503, Code: "busy", RetryAfter: 5 * time.Second, Err: wire.ErrBusy} + sleeps = nil + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{fmt.Errorf("updateIssue: %w", busy), wire.ErrBusy}} + added, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err != nil || !added { + t.Fatalf("addGateWaiter = %v, %v; want added after two busy rounds", added, err) + } + if len(sleeps) != 2 { + t.Fatalf("sleeps = %v, want one per retry", sleeps) + } + if lo, hi := busy.RetryAfter+gateAddWaiterBackoffBase/2, busy.RetryAfter+gateAddWaiterBackoffBase; sleeps[0] < lo || sleeps[0] >= hi { + t.Errorf("wait after Retry-After 5s = %s, want the whole 5s plus the first backoff, in [%s, %s)", sleeps[0], lo, hi) + } + if lo, hi := gateAddWaiterBackoffBase, 2*gateAddWaiterBackoffBase; sleeps[1] < lo || sleeps[1] >= hi { + t.Errorf("wait after a busy answer with no Retry-After = %s, want the plain second backoff, in [%s, %s)", sleeps[1], lo, hi) + } + }) + + t.Run("another SQLSTATE is not retried", func(t *testing.T) { + roles := &contendedGateRoles{fakeGateRoles: newFakeGateRoles(gate()), errs: []error{&sqlStateError{state: "23505"}}} + _, err := addGateWaiter(context.Background(), roles, roles, "bd-g", "b") + if err == nil || len(roles.updates) != 1 { + t.Fatalf("addGateWaiter = %v after %d updates, want one failed write", err, len(roles.updates)) + } + }) +} From 601be8a6bf9b2a93bef0058a1e0a918548f473ed Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 17 Oct 2026 11:21:18 +0000 Subject: [PATCH 08/54] feat(list): bd list's remaining filters over http behind issues.list.filters (#7647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(list): bd list's remaining filters over http behind issues.list.filters Over http every `bd list` filter beyond the created bounds and the title, id and no-assignee scope refused (time bounds, priority range, text and external-ref matches, spec, label pattern/regex, exclude-type, no-parent, no-labels, empty-description, pinned/no-pinned, deferred, overdue, mol/wisp type). The library entry already applied all of them: issueops.Reader.List via workapi.BuildListFilter, shared by the CLI and the handler. This adds only the wire. - Server: GET /v0/beads/issues decodes 29 new optional parameters (listFilterParams, decode only; countIssues' names where it already spells the filter) into the existing ListRequest, behind one new behavior token, issues.list.filters. mol_type/wisp_type are closed enums refused by name (an empty value too). wire_revision stays 2; apigen and the wireshape golden are regenerated (additive only). - Every RFC 3339 bound the listing and count decode is normalized to UTC, so an offset-bearing bound selects what its Z twin does on a zone-less timestamp column. - workapi.BuildListFilter refuses PinnedFlag with NoPinnedFlag (ErrValidation, the CLI's own sentence) instead of letting PinnedFlag win; over http that is a 400 on no_pinned. - Client: the encoder sends the members (encode.ListFiltersMembers, one list read by the encoder, the pre-dial gate and a spec-parity test). Against a server not advertising the token, httpReader.List refuses before dialing with ErrUnsupported naming issues.list.filters. The 29 E-ListRequest ledger refusal rows retire; the ledger doc is updated; skew flag mappings added. - Tests: handler decode/parity/enum/UTC/malformed/pinned-pair tests; encode member/gate/spec parity and bound encoding; per-member ride-behind-token and pre-dial skew refusal; two new Reader contract cases on every leg (dolt, embedded, uow, served): the eight time bounds at exact boundaries (strict, whole-second truncation, NULL never matches, +02:00 equals UTC) and one in/out row per remaining family; three served Reader cases unparked; TestE2E_ListFilterParityOverHTTP (served == embedded, masked server refuses naming the token, no dial); five corpus shapes marked pending until the reference server profile advertises the token. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(list): exclude_status on listIssues, the 30th issues.list.filters member issueops.ListRequest gains ExcludeStatus: statuses to leave out of a listing, appended to the default exclusions (never re-admitting what they hide), comma-separated or repeated, and validated against the workspace's statuses by the same rule CountRequest.ExcludeStatus uses (an unknown name is ErrValidation). The rule moves into one shared function, workapi.ParseExcludeStatus, called by both builders. With AllFlag it reads the active set without enumerating every status, so an http-backed caller can list O(active) rows instead of the whole history. - Server: listIssues decodes a repeatable `exclude_status` under the same issues.list.filters token (no new token); an unknown name is a 400 on exclude_status. apigen and the wireshape golden are regenerated (additive). - Client: the encoder sends it and it joins encode.ListFiltersMembers, so the pre-dial gate asks for the token. - It is refused together with ReadyFlag, like the other members the ready query cannot carry. - No `bd list --exclude-status` flag yet (it would need the CLI docs regenerated); the member is reachable through the library and the wire. Tests: builder unit test (append, AllFlag, comma split, unknown name); handler decode and 400; encoder, round-trip and gate; Reader contract cases on every leg (active set under AllFlag, repeated and comma values, adds to the defaults, narrows an explicit Status, unknown name refused). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(list): label_regex compile errors are 400s; review polish for list filters - An invalid `label_regex` on GET /v0/beads/issues, /v0/beads/ready and /v0/beads/ready:count is now `400 invalid_argument` (`invalid_value` on `label_regex`) instead of `500 internal`. The database still compiles the pattern in its own dialect; a request that sent a regex and earned the engine's "regular expression is invalid" refusal is attributed to the parameter (failRegexReadErr). The same text on a request with no regex stays a server fault. - mol_type and wisp_type refuse each other's vocabulary (tests pin `mol_type=heartbeat` and `wisp_type=swarm`), so the call-site allowlist, not just the spec enum, is covered. - exclude_status with a workspace custom status: leg tests on the embedded store and over the served wire (accepted and applied; a name in no vocabulary refused). ReaderFixture cannot configure custom statuses, so these sit beside the count's twin rather than in the contract. - The search e2e gains a positive arm: --desc-contains returns exactly the row whose description carries the token. - Docs: `all` plus `exclude_status` keeps every status not named (pinned, hooked and custom done/frozen included unless named), not "the active set"; label_regex documents its 400. A corpus source line is made neutral; a test message says 30. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(list): compose issues.list.filters with the --title-contains fold #7631, now on main, folds `bd list --title-contains` onto listIssues `title` and refuses the TitleContains left over from a --title and --title-contains pair that differs, because the listing then had one title parameter. This branch publishes `title_contains` behind issues.list.filters, so that leftover now rides it beside `title` and the server applies both. Neither half is dropped, and against a server without the token the pre-dial gate refuses, naming issues.list.filters. - encode.FoldTitleContains still runs first; its doc now says where a differing pair goes. - Tests: the fold table expects `title` plus `title_contains` for a differing pair, including the ſ/s pair strings.EqualFold would match; the differing-pair refusal names issues.list.filters; the list_filters_test sweep pairs its TitleContains probe with a differing TitleSearch, since a lone one asks only for issues.list.title; the e2e differing-pair refusal becomes a listFilterShapes row that answers exactly child-a over http and embedded, and refuses before dialing without the token. - The gc-polecat-closed-after corpus shape's pending reason also names issues.list.title. The CHANGELOG --title-contains entry no longer lists the differing pair and --closed-after as refused. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/39013611268 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. * fix(list): serve --priority-max in search parity, refuse the parent pair The proxied tier's search parity table still expected a remote `bd search --priority-max 1` to refuse, because listIssues could not carry it. This branch serves priority_max behind issues.list.filters, so the remote route answered and the dolt-race shard went red in TestProxiedServerSearchRoutesAgree. The row now expects the answer on every route. It was the last wireCan=false row; the branch stays, with a comment, for the next flag listIssues cannot carry. - ParentID with NoParent: a served listing now carries no_parent beside parent, and ANDing the two answered only the edge-less rows under the parent's dotted id. BuildListFilter refuses the pair with ErrValidation, in the sentence bd list and BuildCountFilter already use, so issueops.Reader.List refuses it on every backend and the listing answers 400 on no_parent. Pinned by a builder test, a reader conformance case (dolt, embedded, uow, served http) and two httpapi tests. - POST /v0/beads/issues:claimNext shares the ready listing's label_regex but answered a pattern the database cannot compile with 500. It now gives the listing's 400 invalid_value on label_regex, through the helper the listing's mapping is split into. - The descendant walk's ledger rows said listIssues publishes no parameter for members it now serves behind issues.list.filters. They now say the walk does not send them yet, and that --flat or --json asks listIssues directly. The ExcludeStatus, Pinned and ExcludeTypes rows, the parentwalk.go header, the walk table's doc and two walk test comments are corrected the same way, and the ledger doc is regenerated. parentWalkRefusals notes that a member moved off the list must check issues.list.filters itself, because the walk dials past the listing's pre-dial gate. - Spec: the listing's no_parent documents the pair's 400, the count's exclude_status no longer says it has no listing counterpart, and claimNext's label_regex documents its 400. types.gen.go is regenerated. Comment-only: countFilters, and readyScopeFields' ExcludeStatus entry. - CHANGELOG and README say the default `bd list --parent` tree view does not carry the new filters yet; the label_regex entry names claimNext, and a Changed entry records the parent-pair refusal. Second review pass: docs and comments only, no behavior change. - The prose copies of the issues.list.filters member set now match ListFiltersMembers: CapIssuesListFilters and wire.CapListFilters name the status exclusion, and the spec's capabilities text names label-regex. - Every time bound on the listing and the count goes through q.timestamp, which normalizes it to UTC, and sqlbuild compares it strictly at whole-second precision. listIssues' created_before and created_after, and countIssues' created, updated and closed bounds, now say so in the words the new listing bounds use. The count's updated bounds also say what they filter, and its closed bounds say that a NULL closed_at never matches. types.gen.go is regenerated. - README and CHANGELOG: `--flat` or `--json` applies the filters to the parent's direct children, not its whole subtree. - The 29 listRefusal rows this branch serves stay deleted rather than kept as retired rows, as #7550 deleted its TitleSearch, IDFilter and NoAssignee rows when it served them. The CHANGELOG now says the rows are removed, not retired. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../scripts/embedded-storage-test-shards.txt | 3 + CHANGELOG.md | 74 ++- README.md | 12 +- backend/conformance/reader_contract.go | 329 +++++++++ backend/conformance/role_bundle_cases.go | 2 + backend/http/corpus/testdata/corpus.yaml | 27 +- backend/http/e2e_list_filters_test.go | 141 +++- backend/http/e2e_search_test.go | 23 +- cmd/bd/search_routes_parity_test.go | 9 +- engdocs/design/http-divergence-ledger.md | 119 ++-- internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 236 ++++++- internal/httpapi/claim_next.go | 10 +- internal/httpapi/claim_next_test.go | 32 + internal/httpapi/list_filters_test.go | 426 ++++++++++++ internal/httpapi/query.go | 11 +- internal/httpapi/reads.go | 159 ++++- internal/httpapi/reads_test.go | 12 + internal/httpapi/routes.go | 22 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 622 +++++++++++++++++- .../httpapi/wireshape/testdata/golden.json | 271 ++++++++ internal/httpclient/BUILD.bazel | 1 + internal/httpclient/encode/BUILD.bazel | 2 + internal/httpclient/encode/encode.go | 54 ++ internal/httpclient/encode/ledger_fields.go | 69 +- internal/httpclient/encode/list_filters.go | 42 ++ .../httpclient/encode/list_filters_test.go | 134 ++++ internal/httpclient/encode/list_fold.go | 6 +- internal/httpclient/encode/list_fold_test.go | 34 +- internal/httpclient/encode/parentwalk.go | 8 +- internal/httpclient/encode/parentwalk_test.go | 8 +- internal/httpclient/encode/roundtrip_test.go | 36 + internal/httpclient/encode/skew.go | 13 + internal/httpclient/encode/table.go | 71 +- internal/httpclient/list_filters_test.go | 133 ++++ internal/httpclient/list_scope_test.go | 13 +- internal/httpclient/role_reader.go | 13 +- internal/httpclient/roles_test.go | 8 +- internal/httpclient/served_counter_test.go | 17 +- internal/httpclient/served_list_test.go | 17 +- internal/httpclient/served_reader_test.go | 97 ++- internal/httpclient/wire/handshake.go | 13 +- internal/httpclient/wire/vocabulary_test.go | 7 +- internal/storage/dolt/reader_contract_test.go | 12 + .../embeddeddolt/counter_contract_test.go | 52 ++ .../embeddeddolt/reader_contract_test.go | 12 + internal/storage/uow/reader_contract_test.go | 6 + internal/types/types.go | 8 + internal/workapi/count.go | 42 +- internal/workapi/list.go | 24 + internal/workapi/list_pinned_test.go | 72 ++ issueops/counter.go | 8 +- issueops/reader.go | 22 +- issueops/reader_ready_scope.go | 2 + 55 files changed, 3271 insertions(+), 328 deletions(-) create mode 100644 internal/httpapi/list_filters_test.go create mode 100644 internal/httpclient/encode/list_filters.go create mode 100644 internal/httpclient/encode/list_filters_test.go create mode 100644 internal/httpclient/list_filters_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index ac7bf0dbc..c48bdeb79 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -179,6 +179,7 @@ 40 4 TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded 40 4 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering 40 4 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay +40 4 TestEmbeddedReaderListFilterFamiliesNarrowToTheirRows 40 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle 40 5 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState @@ -243,6 +244,7 @@ 40 11 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored 40 12 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +40 12 TestReaderListExcludeStatusAcceptsAWorkspaceCustomStatus 40 12 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne 40 12 TestEmbeddedClaimerRefusesAWispIDAsNotFound 40 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove @@ -444,6 +446,7 @@ 40 29 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest 40 29 TestEmbeddedMergeAndSettleMetadataConflict 40 29 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides +40 29 TestEmbeddedReaderListTimeBoundsAreStrictAndWholeSecond 40 29 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions 40 29 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges 40 29 TestEventsJournalAccessor diff --git a/CHANGELOG.md b/CHANGELOG.md index f6b79d897..99789418d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `bd list`'s remaining filters work over a connected HTTP workspace. + `GET /v0/beads/issues` accepts `updated_after`/`updated_before`, + `closed_after`/`closed_before`, `defer_after`/`defer_before`, + `due_after`/`due_before`, `priority`, `priority_min`, `priority_max`, + `title_contains`, `desc_contains`, `notes_contains`, `external_contains`, + `external_ref`, `spec`, `label_pattern`, `label_regex`, + `empty_description`, `no_labels`, `no_parent`, a repeatable + `exclude_type`, `pinned`, `no_pinned`, `deferred`, `overdue`, `mol_type` + and `wisp_type`, behind one new capability token, `issues.list.filters` + (additive; no wire_revision bump; the names are `issues:count`'s where it + already spells the filter), and applies them through the same Reader role a + local `bd list` reads through. Bounds are strict and compared at + whole-second precision on both routes; relative and date-only values are + resolved on the client's clock and timezone as locally, while `--overdue` + compares against the server's clock. Against a server that does not + advertise the token the client refuses before dialing, naming + `issues.list.filters`, rather than sending a filter an older server rejects + or dropping one and answering a wider listing. The 29 corresponding + divergence-ledger refusal rows are removed. The default `bd list --parent` + tree view does not carry these filters yet: its descendant walk still + refuses one over HTTP rather than drop it, and `--flat` or `--json` skips + the walk and applies them to the parent's direct children (not its whole + subtree). +- `issueops.ListRequest.ExcludeStatus`: statuses to leave out of a listing, + appended to the default exclusions (never re-admitting what they hide), + comma-separated or repeated, each validated against the workspace's + statuses like `CountRequest.ExcludeStatus` (an unknown name is + `ErrValidation`). `GET /v0/beads/issues` accepts it as a repeatable + `exclude_status` under the same `issues.list.filters` token (an unknown + name is a `400` on `exclude_status`), so an HTTP-backed caller can leave + closed history out of an `all` listing (`all` plus + `exclude_status=closed` keeps every status not named, the `pinned` and + `hooked` statuses and custom done/frozen statuses included) instead of + reading the whole history. It is refused together with `ReadyFlag`, like the other + members the ready query cannot carry. There is no `bd list` flag for it + yet. + - `bd serve` publishes the gate members: `await_type`, `await_id`, `timeout` (nanoseconds) and `waiters` on `POST /v0/beads/issues` and on `issues:batchApply` create items, behind the new @@ -377,6 +414,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `bd serve` normalizes every RFC 3339 time bound on the issue listing and + count (`created_*`, `updated_*`, `closed_*`, and the new `defer_*` and + `due_*`) to UTC before it reaches storage. A bound carrying an offset + (`...+02:00`) now selects exactly what its `Z` twin does; previously a + backend whose timestamp column has no zone could read the offset bound's + wall clock and compare it hours off. `bd`'s own client always sent `Z`, so + its answers are unchanged. +- `bd serve` answers a `label_regex` the database cannot compile with + `400 invalid_argument` (`invalid_value` on `label_regex`) on + `GET /v0/beads/issues`, `GET /v0/beads/ready`, `GET /v0/beads/ready:count` + and `POST /v0/beads/issues:claimNext`, instead of `500 internal`. The pattern is + still compiled by the database in its own dialect; only the attribution + of its refusal changed. +- `issueops.Reader.List` refuses `PinnedFlag` together with `NoPinnedFlag` + with `ErrValidation` (`--pinned and --no-pinned are mutually exclusive`) on + every backend; it used to let `PinnedFlag` win silently. `bd list` already + refused the pair at flag parse, so its behavior is unchanged; over HTTP the + pair is a `400 invalid_argument` on `no_pinned`. +- `issueops.Reader.List` refuses `ParentID` together with `NoParent` with + `ErrValidation` (`--parent and --no-parent are mutually exclusive`) on + every backend, as `CountRequest` already did; it used to answer with only + the edge-less rows under the parent's dotted id, which both clauses match. + `bd list` already refused the pair at flag parse, so its behavior is + unchanged; over HTTP the pair is a `400 invalid_argument` on `no_parent`. + - **`bd gate discover` skips template gates, as `bd gate check` does.** A template gh:run gate is a proto that work is poured from; on the direct, proxied and embedded routes `bd gate discover` used to record a discovered @@ -636,12 +698,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `bd list --title-contains X` works against an HTTP workspace. The client folds it onto the listing's existing `title` parameter (the store compiles both flags to the same case-insensitive substring predicate), so it needs - the `issues.list.title` capability, like `--title`. Over HTTP these still - refuse: a `--title` and a `--title-contains` that differ other than in - letter case (the listing has one title parameter, and dropping either value - would widen the answer); either flag with `--parent` in the default tree - view; and `--closed-after`, so gascity's polecat pre-flight lookup of - recently closed beads still fails. + the `issues.list.title` capability, like `--title`. A `--title` and a + `--title-contains` that differ other than in letter case are two + predicates, so they ride `title` and `title_contains` together and also + need `issues.list.filters` (see Added), as does the `--closed-after` in + gascity's polecat pre-flight lookup of recently closed beads. Either flag + with `--parent` in the default tree view still refuses over HTTP. - **`bd gate create`, `bd gate add-waiter` and `bd gate discover` work over a connected HTTP workspace.** They wrote through the legacy `CreateIssue`, `AddDependency` and map-based `UpdateIssue`, which the HTTP store does not diff --git a/README.md b/README.md index 5760ce630..5faca32f1 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,17 @@ too, each in its own way. `bd search` asks the server's listing with its text (` refuses before dialing against a server that does not advertise it. `bd list --no-assignee`, `--title` and `--id` likewise ride the listing's `no_assignee`, `title` and repeatable `id`, each behind its own capability -(`issues.list.noAssignee`, `issues.list.title`, `issues.list.ids`). +(`issues.list.noAssignee`, `issues.list.title`, `issues.list.ids`), and the +rest of `bd list`'s filters (`--updated-*`, `--closed-*`, `--defer-*`, +`--due-*`, `--priority*`, the text and external-ref matches, `--spec`, +`--label-pattern`/`--label-regex`, `--exclude-type`, `--no-parent`, +`--no-labels`, `--empty-description`, `--pinned`/`--no-pinned`, +`--deferred`, `--overdue`, `--mol-type`, `--wisp-type`) ride it behind one +capability, `issues.list.filters`. Time bounds are resolved on the client as +they are locally; `--overdue` compares against the server's clock. The +default `--parent` tree view does not carry those filters yet and refuses one +rather than drop it; `--flat` or `--json` applies them to the parent's direct +children (not its whole subtree). `bd show --current` finds the actor's in-progress or hooked issue through the server's boolean query. `bd recompute-blocked` is a no-op that exits 0: the server maintains `is_blocked` on every write, and repairing a stale flag is diff --git a/backend/conformance/reader_contract.go b/backend/conformance/reader_contract.go index 2f93f818c..245a9efaa 100644 --- a/backend/conformance/reader_contract.go +++ b/backend/conformance/reader_contract.go @@ -3175,6 +3175,335 @@ func assertReaderBriefRow(t *testing.T, what string, brief, full *types.Issue) { } } +// RunReaderListTimeBoundsAreStrictAndWholeSecond pins the eight time bounds +// ListRequest carries beside the created pair — UpdatedAfter/Before, +// ClosedAfter/Before, DeferAfter/Before and DueAfter/Before — at their exact +// boundaries (reader.go: each is a strict bound on its column). +// +// THE BOUNDARY IS THE ASSERTION. Three rows sit one second apart around an +// instant T, so a body that compared inclusively, swapped the two ends, or +// bound the wrong column fails on a row rather than on nothing: +// +// - after=T selects exactly the T+1s row and before=T exactly the T-1s row, +// so the row AT T is excluded by both; +// - a bound carrying a sub-second remainder (T+900ms) behaves as T on both +// ends, because the bound is compared at whole-second precision (the +// storage builder formats it as RFC 3339 without a fraction); +// - a row whose column is NULL — an open row's closed_at, an unset +// defer_until or due_at — never matches either bound; +// - a bound spelled in a non-UTC zone selects exactly what the same instant +// spelled in UTC does. An offset is part of the instant, not decoration, +// and a body that dropped it would move the bound by the offset. +// +// Every family scopes itself by id and seeds no labels, so no write after the +// insert can move updated_at off the instant it was seeded at; AllFlag keeps +// the closed rows in view. +func RunReaderListTimeBoundsAreStrictAndWholeSecond(t *testing.T, ctx context.Context, fixture ReaderFixture) { + t.Helper() + at := time.Now().UTC().Truncate(time.Second).Add(-3 * time.Hour) + created := at.Add(-time.Hour) + instants := []struct { + tag string + at time.Time + }{ + {"early", at.Add(-time.Second)}, + {"exact", at}, + {"late", at.Add(time.Second)}, + } + + for _, family := range []struct { + name string + // set stamps the family's column onto a seed. + set func(issue *types.Issue, v time.Time) + // nullable seeds a fourth row with the column NULL. + nullable bool + after func(*publicops.ListRequest, *time.Time) + before func(*publicops.ListRequest, *time.Time) + }{ + { + name: "updated", + set: func(issue *types.Issue, v time.Time) { issue.UpdatedAt = v }, + after: func(r *publicops.ListRequest, v *time.Time) { r.UpdatedAfter = v }, + before: func(r *publicops.ListRequest, v *time.Time) { r.UpdatedBefore = v }, + }, + { + name: "closed", + set: func(issue *types.Issue, v time.Time) { + issue.Status = types.StatusClosed + issue.UpdatedAt = v + issue.ClosedAt = &v + }, + nullable: true, + after: func(r *publicops.ListRequest, v *time.Time) { r.ClosedAfter = v }, + before: func(r *publicops.ListRequest, v *time.Time) { r.ClosedBefore = v }, + }, + { + name: "defer", + set: func(issue *types.Issue, v time.Time) { issue.DeferUntil = &v }, + nullable: true, + after: func(r *publicops.ListRequest, v *time.Time) { r.DeferAfter = v }, + before: func(r *publicops.ListRequest, v *time.Time) { r.DeferBefore = v }, + }, + { + name: "due", + set: func(issue *types.Issue, v time.Time) { issue.DueAt = &v }, + nullable: true, + after: func(r *publicops.ListRequest, v *time.Time) { r.DueAfter = v }, + before: func(r *publicops.ListRequest, v *time.Time) { r.DueBefore = v }, + }, + } { + caseTag := "ltb" + family.name + ids := map[string]string{} + var scope []string + for _, inst := range instants { + id := readerID(fixture, caseTag, inst.tag) + ids[inst.tag] = id + scope = append(scope, id) + issue := readerIssue(id, types.TypeTask, "") + issue.CreatedAt, issue.UpdatedAt = created, created + family.set(issue, inst.at) + seedReaderIssue(t, ctx, fixture, issue) + } + if family.nullable { + id := readerID(fixture, caseTag, "null") + scope = append(scope, id) + issue := readerIssue(id, types.TypeTask, "") + issue.CreatedAt, issue.UpdatedAt = created, created + seedReaderIssue(t, ctx, fixture, issue) + } + idScope := readerIDFilter(scope...) + + list := func(what string, apply func(*publicops.ListRequest, *time.Time), bound time.Time) []string { + t.Helper() + req := publicops.ListRequest{IDFilter: idScope, AllFlag: true} + apply(&req, &bound) + page, err := fixture.Reader.List(ctx, req) + if err != nil { + t.Fatalf("List (%s %s): %v", family.name, what, err) + } + assertReaderPageNotNil(t, "List ("+family.name+" "+what+")", page) + got := readerPageIDs(page) + slices.Sort(got) + return got + } + expect := func(what string, apply func(*publicops.ListRequest, *time.Time), bound time.Time, want string) { + t.Helper() + if got := list(what, apply, bound); !slices.Equal(got, []string{want}) { + t.Errorf("List (%s %s %s) = %v, want exactly [%s]", family.name, what, bound.Format(time.RFC3339Nano), got, want) + } + } + + plus2 := time.FixedZone("UTC+2", 2*60*60) + for _, bound := range []struct { + what string + v time.Time + }{ + {"at T", at}, + {"at T with a sub-second remainder", at.Add(900 * time.Millisecond)}, + {"at T spelled in a +02:00 zone", at.In(plus2)}, + } { + expect("after "+bound.what, family.after, bound.v, ids["late"]) + expect("before "+bound.what, family.before, bound.v, ids["early"]) + } + + // The NULL row is outside even the widest bounds on both ends. + if family.nullable { + wide := at.Add(-24 * time.Hour) + if got, want := list("after a day earlier", family.after, wide), sortedCopy(ids["early"], ids["exact"], ids["late"]); !slices.Equal(got, want) { + t.Errorf("List (%s after a day earlier) = %v, want %v — a NULL column never matches", family.name, got, want) + } + if got, want := list("before a day later", family.before, at.Add(24*time.Hour)), sortedCopy(ids["early"], ids["exact"], ids["late"]); !slices.Equal(got, want) { + t.Errorf("List (%s before a day later) = %v, want %v — a NULL column never matches", family.name, got, want) + } + } + } +} + +// RunReaderListFilterFamiliesNarrowToTheirRows pins one included and one +// excluded row for each remaining ListRequest filter family that `bd list` +// publishes beyond the scope filters: the priority trio, the three text +// substrings, the empty-description and no-labels predicates, the top-level +// predicate, the type exclusion, the spec prefix, the label pattern and +// regex, the external-ref pair, the pinned-flag pair, the deferred and +// overdue predicates, the status exclusion and the molecule type. +// +// Each row is scoped by id and every arm asks AllFlag (the pinned arms +// excepted, which are about that interplay), so an arm whose filter the body +// dropped answers its whole scope and fails, and one whose filter the body +// inverted answers the other row. +// +// PinnedFlag with NoPinnedFlag is a contradiction, and the role refuses it as +// a validation error rather than letting one of the two silently win. +// ParentID with NoParent is refused the same way, as CountRequest refuses it. +func RunReaderListFilterFamiliesNarrowToTheirRows(t *testing.T, ctx context.Context, fixture ReaderFixture) { + t.Helper() + id := func(name string) string { return readerID(fixture, "lff", name) } + seed := func(name string, mutate func(*types.Issue)) string { + t.Helper() + issue := readerIssue(id(name), types.TypeTask, "") + if mutate != nil { + mutate(issue) + } + seedReaderIssue(t, ctx, fixture, issue) + return issue.ID + } + intp := func(v int) *int { return &v } + past := time.Now().UTC().Truncate(time.Second).Add(-48 * time.Hour) + future := time.Now().UTC().Truncate(time.Second).Add(48 * time.Hour) + label := readerLabel(fixture, "lffpat") + external := fixture.IssuePrefix + "-LFF-ext-17" + spec := "docs/" + fixture.IssuePrefix + "-lff/" + + p0 := seed("p0", func(i *types.Issue) { i.Priority = 0 }) + p2 := seed("p2", func(i *types.Issue) { i.Priority = 2 }) + p4 := seed("p4", func(i *types.Issue) { i.Priority = 4 }) + + textT := seed("text-t", func(i *types.Issue) { i.Title = "a title carrying ttok-lff" }) + textD := seed("text-d", func(i *types.Issue) { i.Description = "a description carrying dtok-lff" }) + textN := seed("text-n", func(i *types.Issue) { i.Notes = "notes carrying ntok-lff" }) + + labeled := seed("labeled", func(i *types.Issue) { i.Labels = []string{label} }) + unlabeled := seed("unlabeled", nil) + + parent := seed("parent", func(i *types.Issue) { i.IssueType = types.TypeEpic }) + child := seed("child", nil) + if err := fixture.AddDependency(ctx, &types.Dependency{IssueID: child, DependsOnID: parent, Type: types.DepParentChild}, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", child, parent, err) + } + loose := seed("loose", nil) + + typeTask := seed("type-task", nil) + typeBug := seed("type-bug", func(i *types.Issue) { i.IssueType = types.TypeBug }) + typeChore := seed("type-chore", func(i *types.Issue) { i.IssueType = types.TypeChore }) + + specced := seed("specced", func(i *types.Issue) { i.SpecID = spec + "design.md" }) + unspecced := seed("unspecced", nil) + + extRow := seed("ext", func(i *types.Issue) { i.ExternalRef = &external }) + extOther := seed("ext-other", func(i *types.Issue) { + other := fixture.IssuePrefix + "-other-99" + i.ExternalRef = &other + }) + + pinned := seed("pinned", func(i *types.Issue) { i.Pinned = true }) + unpinned := seed("unpinned", nil) + + deferred := seed("deferred", func(i *types.Issue) { i.DeferUntil = &future }) + undeferred := seed("undeferred", nil) + + overdue := seed("overdue", func(i *types.Issue) { i.DueAt = &past }) + dueLater := seed("due-later", func(i *types.Issue) { i.DueAt = &future }) + closedOverdue := seed("closed-overdue", func(i *types.Issue) { + i.Status = types.StatusClosed + closedAt := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + i.ClosedAt = &closedAt + i.DueAt = &past + }) + noDue := seed("no-due", nil) + + swarm := seed("mol-swarm", func(i *types.Issue) { i.MolType = types.MolTypeSwarm }) + patrol := seed("mol-patrol", func(i *types.Issue) { i.MolType = types.MolTypePatrol }) + + stOpen := seed("st-open", nil) + stProgress := seed("st-progress", func(i *types.Issue) { i.Status = types.StatusInProgress }) + stBlocked := seed("st-blocked", func(i *types.Issue) { i.Status = types.StatusBlocked }) + stClosed := seed("st-closed", func(i *types.Issue) { + i.Status = types.StatusClosed + closedAt := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + i.ClosedAt = &closedAt + }) + statusScope := readerIDFilter(stOpen, stProgress, stBlocked, stClosed) + + molType := func(m types.MolType) *publicops.MolType { return &m } + scoped := func(ids ...string) publicops.ListRequest { + return publicops.ListRequest{IDFilter: readerIDFilter(ids...), AllFlag: true} + } + with := func(req publicops.ListRequest, mutate func(*publicops.ListRequest)) publicops.ListRequest { + mutate(&req) + return req + } + + for _, test := range []struct { + name string + req publicops.ListRequest + want []string + }{ + {"Priority is exact", with(scoped(p0, p2, p4), func(r *publicops.ListRequest) { r.Priority = intp(2) }), []string{p2}}, + {"Priority 0 is a value, not unset", with(scoped(p0, p2, p4), func(r *publicops.ListRequest) { r.Priority = intp(0) }), []string{p0}}, + {"PriorityMin is inclusive", with(scoped(p0, p2, p4), func(r *publicops.ListRequest) { r.PriorityMin = intp(2) }), []string{p2, p4}}, + {"PriorityMax is inclusive", with(scoped(p0, p2, p4), func(r *publicops.ListRequest) { r.PriorityMax = intp(2) }), []string{p0, p2}}, + {"PriorityMin and PriorityMax compose", with(scoped(p0, p2, p4), func(r *publicops.ListRequest) { r.PriorityMin, r.PriorityMax = intp(1), intp(3) }), []string{p2}}, + + {"TitleContains is a case-insensitive substring", with(scoped(textT, textD, textN), func(r *publicops.ListRequest) { r.TitleContains = "TTOK-LFF" }), []string{textT}}, + {"DescContains reads the description only", with(scoped(textT, textD, textN), func(r *publicops.ListRequest) { r.DescContains = "dtok-lff" }), []string{textD}}, + {"NotesContains reads the notes only", with(scoped(textT, textD, textN), func(r *publicops.ListRequest) { r.NotesContains = "ntok-lff" }), []string{textN}}, + {"EmptyDesc keeps the rows with no description", with(scoped(textT, textD, textN), func(r *publicops.ListRequest) { r.EmptyDesc = true }), []string{textT, textN}}, + + {"NoLabels keeps the unlabeled row", with(scoped(labeled, unlabeled), func(r *publicops.ListRequest) { r.NoLabels = true }), []string{unlabeled}}, + {"LabelPattern is a glob", with(scoped(labeled, unlabeled), func(r *publicops.ListRequest) { r.LabelPattern = fixture.IssuePrefix + "-scope-lff*" }), []string{labeled}}, + {"LabelRegex is a regular expression", with(scoped(labeled, unlabeled), func(r *publicops.ListRequest) { r.LabelRegex = "-scope-lffp.t$" }), []string{labeled}}, + + {"NoParent keeps the top-level rows", with(scoped(parent, child, loose), func(r *publicops.ListRequest) { r.NoParent = true }), []string{parent, loose}}, + + {"ExcludeTypes drops the named type", with(scoped(typeTask, typeBug, typeChore), func(r *publicops.ListRequest) { r.ExcludeTypes = []string{"bug"} }), []string{typeTask, typeChore}}, + {"ExcludeTypes takes a comma-separated value", with(scoped(typeTask, typeBug, typeChore), func(r *publicops.ListRequest) { r.ExcludeTypes = []string{"bug,chore"} }), []string{typeTask}}, + + {"SpecPrefix is a prefix", with(scoped(specced, unspecced), func(r *publicops.ListRequest) { r.SpecPrefix = spec }), []string{specced}}, + + {"ExternalContains is a case-insensitive substring", with(scoped(extRow, extOther), func(r *publicops.ListRequest) { r.ExternalContains = "lff-EXT" }), []string{extRow}}, + {"ExternalRef is exact", with(scoped(extRow, extOther), func(r *publicops.ListRequest) { r.ExternalRef = external }), []string{extRow}}, + + {"AllFlag alone admits both pinned-flag states", scoped(pinned, unpinned), []string{pinned, unpinned}}, + {"PinnedFlag selects the flagged row", publicops.ListRequest{IDFilter: readerIDFilter(pinned, unpinned), PinnedFlag: true}, []string{pinned}}, + {"NoPinnedFlag holds the unflagged predicate under AllFlag", with(scoped(pinned, unpinned), func(r *publicops.ListRequest) { r.NoPinnedFlag = true }), []string{unpinned}}, + + {"DeferredFlag keeps the deferred row", with(scoped(deferred, undeferred), func(r *publicops.ListRequest) { r.DeferredFlag = true }), []string{deferred}}, + + {"OverdueFlag keeps the open row past its due date", with(scoped(overdue, dueLater, closedOverdue, noDue), func(r *publicops.ListRequest) { r.OverdueFlag = true }), []string{overdue}}, + + {"ExcludeStatus under AllFlag keeps every status not named", with(scoped(stOpen, stProgress, stBlocked, stClosed), func(r *publicops.ListRequest) { r.ExcludeStatus = []string{"closed"} }), []string{stOpen, stProgress, stBlocked}}, + {"ExcludeStatus takes repeated and comma-separated values", with(scoped(stOpen, stProgress, stBlocked, stClosed), func(r *publicops.ListRequest) { r.ExcludeStatus = []string{"blocked,in_progress", " closed "} }), []string{stOpen}}, + {"ExcludeStatus adds to the default exclusions rather than replacing them", publicops.ListRequest{IDFilter: statusScope, ExcludeStatus: []string{"blocked"}}, []string{stOpen, stProgress}}, + {"ExcludeStatus narrows an explicit Status", publicops.ListRequest{IDFilter: statusScope, Status: "open,blocked,closed", ExcludeStatus: []string{"blocked"}}, []string{stOpen, stClosed}}, + + {"MolType is exact", with(scoped(swarm, patrol), func(r *publicops.ListRequest) { r.MolType = molType(types.MolTypeSwarm) }), []string{swarm}}, + } { + page, err := fixture.Reader.List(ctx, test.req) + if err != nil { + t.Errorf("List (%s): %v", test.name, err) + continue + } + assertReaderPageIDSet(t, "List ("+test.name+")", page, test.want) + } + + // An exclusion naming no status in the workspace vocabulary is refused, + // never an exclusion that silently excludes nothing. + if _, err := fixture.Reader.List(ctx, publicops.ListRequest{IDFilter: statusScope, ExcludeStatus: []string{"clsoed"}}); !errors.Is(err, publicops.ErrValidation) { + t.Errorf("List with ExcludeStatus naming an unknown status = %v, want an error wrapping ErrValidation", err) + } + + _, err := fixture.Reader.List(ctx, publicops.ListRequest{ + IDFilter: readerIDFilter(pinned, unpinned), PinnedFlag: true, NoPinnedFlag: true, + }) + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("List with PinnedFlag and NoPinnedFlag = %v, want an error wrapping ErrValidation", err) + } + + _, err = fixture.Reader.List(ctx, publicops.ListRequest{ + IDFilter: readerIDFilter(parent, child, loose), ParentID: parent, NoParent: true, + }) + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("List with ParentID and NoParent = %v, want an error wrapping ErrValidation", err) + } +} + +// sortedCopy returns its arguments as a sorted slice. +func sortedCopy(ids ...string) []string { + out := slices.Clone(ids) + slices.Sort(out) + return out +} + // readerIssue builds the seed every case starts from: an open, unassigned, // unblocked task that qualifies for ready work. A case that needs it to fail one // of those tests changes the field it means to test and nothing else. diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index d000dedb6..1c5e2cf00 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -604,6 +604,8 @@ var roleContractCases = []roleContract{ RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt, RunReaderListBriefDropsTheFreeFormTextAndNothingElse, RunReaderReadyBriefDropsTheFreeFormTextAndNothingElse, + RunReaderListTimeBoundsAreStrictAndWholeSecond, + RunReaderListFilterFamiliesNarrowToTheirRows, ), roleCases("ReadyClaimer", "ReadyClaimer() and IssueReader()", oncePerRole, diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index ad989aa79..788ea038f 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -315,7 +315,32 @@ shapes: - id: gc-polecat-title-contains-closed-after source: gascity core formulas/mol-polecat-base.toml (pre-flight 3d, list --title-contains --status closed --closed-after DATE --json) args: [list, --title-contains, "closed task", --status, closed, --closed-after, "2000-01-01", --json] - pending: {slice: "SRV-1+list-closed-after", reason: "two refusals: against the reference server --title-contains folds onto listIssues title behind issues.list.title, which it does not advertise, so the client refuses before dialing; past that, listIssues publishes no closed_after parameter, so the encoder refuses --closed-after (E-ListRequest.ClosedAfter); remove the mark when reference_server.json is upgraded and listIssues carries closed_after"} + pending: {slice: "SRV-1", reason: "needs issues.list.title (--title-contains folds onto listIssues title) and issues.list.filters (closed_after), which the reference server does not advertise, so the client refuses before dialing; remove the mark when reference_server.json is upgraded"} + - id: gc-polecat-closed-after + source: gascity core formulas/mol-polecat-base.toml:267 (list --title-contains --status closed --closed-after --json) + args: [list, --title-contains, "task", --status, closed, --closed-after, "2000-01-01", --json] + compare: ids + pending: {slice: "SRV-1", reason: "needs issues.list.title (--title-contains folds onto listIssues title) and issues.list.filters (closed_after), which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListFilterParityOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: wf-session-cleanup + source: an order cleanup script's closed-session sweep (list --all --include-infra --include-gates --status closed --label --closed-before --updated-before) + args: [list, --json, --all, --include-infra, --include-gates, --status, closed, --label, "lane:a", --closed-before, "2100-01-01T00:00:00Z", --updated-before, "2100-01-01T00:00:00Z", --limit, "0"] + compare: ids + pending: {slice: "SRV-1", reason: "needs issues.list.filters, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListFilterParityOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: list-filter-priority-range + source: bd list --priority-min/--priority-max (listing filter parity) + args: [list, --json, --priority-min, "1", --priority-max, "3", --limit, "0"] + compare: ids + pending: {slice: "SRV-1", reason: "needs issues.list.filters, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListFilterParityOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: list-filter-exclude-type-no-parent + source: bd list --exclude-type --no-parent (listing filter parity) + args: [list, --json, --exclude-type, epic, --no-parent, --limit, "0"] + compare: ids + pending: {slice: "SRV-1", reason: "needs issues.list.filters, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListFilterParityOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: list-filter-updated-after + source: bd list --updated-after (listing filter parity) + args: [list, --json, --updated-after, "2000-01-01", --limit, "0"] + compare: ids + pending: {slice: "SRV-1", reason: "needs issues.list.filters, which the reference server does not advertise, so the client refuses before dialing (pinned passing against the current server by backend/http TestE2E_ListFilterParityOverHTTP); remove the mark when reference_server.json is upgraded"} - id: mc-list-assignee-status source: mc review summaries (list --assignee --json --limit --status) args: [list, --assignee, alice, --json, --limit, "10", --status, open] diff --git a/backend/http/e2e_list_filters_test.go b/backend/http/e2e_list_filters_test.go index cd26c97ec..c9d550c50 100644 --- a/backend/http/e2e_list_filters_test.go +++ b/backend/http/e2e_list_filters_test.go @@ -101,19 +101,6 @@ func TestE2E_ListScopeFiltersOverHTTP(t *testing.T) { }) } - // The listing publishes one title parameter: a differing pair refuses - // rather than drop either half. - t.Run("differing title pair refuses", func(t *testing.T) { - args := []string{"list", "--all", "--title", "child-a", "--title-contains", "child-b", "--json"} - r := remote.run(nil, args...) - if r.code == 0 { - t.Fatalf("http: bd %s exited 0; one half of the pair was dropped: %s", strings.Join(args, " "), r.stdout) - } - if !strings.Contains(r.stdout+r.stderr, "TitleContains") { - t.Errorf("the refusal does not name TitleContains: stdout=%s stderr=%s", r.stdout, r.stderr) - } - }) - t.Run("MC server mask refuses before dialing", func(t *testing.T) { proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) f := newHTTPFixture(t, bin, proxyAddr) @@ -137,3 +124,131 @@ func TestE2E_ListScopeFiltersOverHTTP(t *testing.T) { } }) } + +// listFilterShapes are the `bd list` shapes that ride the listing filters +// behind issues.list.filters, each with the titles it must answer on the seed +// plus seedListFilters. Five are corpus shapes and one pins the title pair the +// fold leaves: +// - wf-session-cleanup: the session-cleanup sweep's closed-and-stale scan; +// - gc-polecat-closed-after: the polecat formula's "has the fix landed" +// check (title substring, closed after a cutoff); +// - list-filter-priority-range, list-filter-exclude-type-no-parent and +// list-filter-updated-after: one representative per remaining family; +// - title-and-title-contains-differ (not a corpus shape): a `--title` and a +// `--title-contains` that differ ride `title` and `title_contains` +// together, and the server applies both. +var listFilterShapes = []struct { + corpusID string + args []string + want []string +}{ + {"wf-session-cleanup", + []string{"list", "--json", "--all", "--include-infra", "--include-gates", "--status", "closed", "--label", "lane:a", + "--closed-before", "2100-01-01T00:00:00Z", "--updated-before", "2100-01-01T00:00:00Z", "--limit", "0"}, + []string{"lane task closed"}}, + {"gc-polecat-closed-after", + []string{"list", "--title-contains", "task", "--status", "closed", "--closed-after", "2000-01-01", "--json"}, + []string{"lane task closed", "other lane task closed"}}, + {"list-filter-priority-range", + []string{"list", "--json", "--priority-min", "1", "--priority-max", "3", "--limit", "0"}, + []string{"child-a", "child-b", "loner", "open task p2", "parent", "tmpl"}}, + {"list-filter-exclude-type-no-parent", + []string{"list", "--json", "--exclude-type", "epic", "--no-parent", "--limit", "0"}, + []string{"loner", "open task p2", "p4 row", "tmpl"}}, + {"list-filter-updated-after", + []string{"list", "--json", "--updated-after", "2000-01-01", "--limit", "0"}, + []string{"child-a", "child-b", "child-x", "loner", "open task p2", "p4 row", "parent", "tmpl"}}, + // Either substring alone answers more: "child" every child, "a" the + // lane tasks, parent and open task p2 too. + {"title-and-title-contains-differ", + []string{"list", "--all", "--title", "child", "--title-contains", "a", "--json"}, + []string{"child-a"}}, +} + +// seedListFilters adds the rows the filter shapes split: two closed tasks on +// different lanes (the label keeps one out of the session-cleanup scan, the +// title substring keeps child-c out of the polecat check), a P4 row the +// priority range leaves out, and an open P2 task beside it. +func (f *remoteRolesFixture) seedListFilters() { + f.t.Helper() + a := f.create("lane task closed", "-t", "task", "-l", "lane:a") + b := f.create("other lane task closed", "-t", "task", "-l", "lane:b") + f.mustRun(nil, "close", a, "--reason", "done") + f.mustRun(nil, "close", b, "--reason", "done") + f.create("p4 row", "-t", "task", "-p", "4") + f.create("open task p2", "-t", "task", "-p", "2") +} + +// withoutListFilters rewrites the handshake into a server that is current in +// every respect but one: it does not advertise issues.list.filters. +func withoutListFilters(body map[string]any) { + caps, _ := body["capabilities"].([]any) + kept := make([]any, 0, len(caps)) + for _, c := range caps { + if c != "issues.list.filters" { + kept = append(kept, c) + } + } + body["capabilities"] = kept +} + +// TestE2E_ListFilterParityOverHTTP: `bd list`'s time bounds and every other +// filter the listing refused over http now ride listIssues behind the one +// behavior token issues.list.filters, applied by the same Reader role a local +// `bd list` reads through. Each shape must answer the rows an embedded +// workspace answers, and exactly the rows it names. +// +// Against a server that does not advertise the token each shape refuses +// BEFORE dialing listIssues, naming the token, rather than answering a wider +// listing. +func TestE2E_ListFilterParityOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + remote := newHTTPFixture(t, bin, addr) + local := newEmbeddedFixture(t, bin) + for _, f := range []*remoteRolesFixture{remote, local} { + f.seed() + f.seedListFilters() + } + + for _, shape := range listFilterShapes { + t.Run(shape.corpusID, func(t *testing.T) { + r := remote.run(nil, shape.args...) + if r.code != 0 { + t.Fatalf("http: bd %s failed (exit %d): stderr=%s", strings.Join(shape.args, " "), r.code, r.stderr) + } + l := local.mustRun(nil, shape.args...) + got, want := remote.titlesOf(r.stdout), local.titlesOf(l.stdout) + if !slices.Equal(got, want) { + t.Errorf("http rows %v, embedded rows %v", got, want) + } + if !slices.Equal(got, shape.want) { + t.Errorf("http rows %v, want exactly %v", got, shape.want) + } + }) + } + + t.Run("a server without issues.list.filters refuses before dialing", func(t *testing.T) { + proxyAddr, counter := startMaskingProxy(t, addr, withoutListFilters) + f := newHTTPFixture(t, bin, proxyAddr) + f.ids = remote.ids // same workspace behind the proxy + for _, shape := range listFilterShapes { + counter.reset() + r := f.run(nil, shape.args...) + if r.code == 0 { + t.Errorf("%s: bd %s exited 0 against a server lacking issues.list.filters: %s", shape.corpusID, strings.Join(shape.args, " "), r.stdout) + continue + } + if !strings.Contains(r.stderr+r.stdout, "issues.list.filters") { + t.Errorf("%s: the refusal does not name issues.list.filters: stdout=%s stderr=%s", shape.corpusID, r.stdout, r.stderr) + } + for _, p := range counter.snapshot() { + if p == "GET /v0/beads/issues" { + t.Errorf("%s: dialed listIssues against a server lacking issues.list.filters: %v", shape.corpusID, counter.snapshot()) + break + } + } + } + }) +} diff --git a/backend/http/e2e_search_test.go b/backend/http/e2e_search_test.go index b932fc71a..9ae814566 100644 --- a/backend/http/e2e_search_test.go +++ b/backend/http/e2e_search_test.go @@ -49,9 +49,24 @@ func TestE2E_SearchOverHTTP(t *testing.T) { t.Errorf("bd search %s did not match the issue by its id: %v", other, byID) } - // A flag listIssues publishes nothing for refuses rather than widening the - // search. - if r := bd("search", "routed", "--desc-contains", "pool", "--json"); r.code == 0 { - t.Errorf("bd search --desc-contains succeeded over http; the wire cannot carry it, so it must refuse: %s", r.stdout) + // A search filter rides listIssues behind issues.list.filters and narrows + // rather than being dropped: "pool" is in a TITLE here and in no + // description, so the answer is empty, where a dropped filter would answer + // every routed row. + r = bd("search", "routed", "--desc-contains", "pool", "--json") + if r.code != 0 { + t.Fatalf("bd search --desc-contains failed over http (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + if narrowed := jsonIDSet(t, r.stdout); len(narrowed) != 0 { + t.Errorf("bd search routed --desc-contains pool = %v, want nothing: no description contains it", narrowed) + } + + // And it matches: a description carrying the token answers exactly that + // row, so the empty answer above is the filter narrowing, not a filter + // that matches nothing for some other reason. + mustRun(t, bd, "update", alsoRouted, "--description", "a lantern in the description", "--json") + matched := jsonIDSet(t, mustRun(t, bd, "search", "routed", "--desc-contains", "LANTERN", "--json").stdout) + if len(matched) != 1 || !matched[alsoRouted] { + t.Errorf("bd search routed --desc-contains LANTERN = %v, want exactly %s, whose description contains it", matched, alsoRouted) } } diff --git a/cmd/bd/search_routes_parity_test.go b/cmd/bd/search_routes_parity_test.go index af851f54d..4ce48a5bf 100644 --- a/cmd/bd/search_routes_parity_test.go +++ b/cmd/bd/search_routes_parity_test.go @@ -97,7 +97,7 @@ var searchRouteQueries = []searchRouteQuery{ {"sort priority reverse", []string{"routed", "--sort", "priority", "--reverse"}, true, true, []string{srDone, srGamma, srBeta, srAlpha}}, {"query flag", []string{"--query", "gamma"}, false, true, []string{srGamma}}, {"no match", []string{"nothing-matches-this"}, false, true, nil}, - {"priority-max", []string{"routed", "--priority-max", "1"}, false, false, []string{srAlpha, srBeta}}, + {"priority-max", []string{"routed", "--priority-max", "1"}, false, true, []string{srAlpha, srBeta}}, // --type is expanded as `bd list` expands it before the Reader validates it. {"type alias", []string{"routed", "--type", "feat"}, false, true, []string{srGamma}}, // Every kind of row comes back under no flag and under its own --type. An @@ -193,8 +193,11 @@ func searchRouteComparable(rows []searchRouteRow, ordered bool) []string { // checkSearchRoute runs every query on one route and holds each answer to its // want. A query listIssues cannot carry (wireCan false) must refuse on a // remote route rather than answer without the flag, and every refusal must -// refuse on every route. It returns the decoded answers by query name, for a -// route that is compared row for row with another one over the same database. +// refuse on every route. No query sets wireCan false today: issues.list.filters +// put the last flag search had to refuse, --priority-max, on the wire. The +// branch stays for the next flag listIssues cannot carry. It returns the +// decoded answers by query name, for a route that is compared row for row with +// another one over the same database. func checkSearchRoute(t *testing.T, route, bd, dir string, env []string, remote bool) map[string][]searchRouteRow { t.Helper() answers := map[string][]searchRouteRow{} diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 269d45eda..0e424adff 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -263 rows: 201 refuse, 35 degrade, 27 retired. +234 rows: 172 refuse, 35 degrade, 27 retired. ## Whole-behavior divergences (D9 ledger) @@ -180,35 +180,6 @@ The listing publishes far more filters than the v0 wire does; every one the wire | ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | |---|---|---|---|---|---|---| -| E-ListRequest.SpecPrefix | refuse | `ListRequest.SpecPrefix` | a spec-id prefix restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.LabelPattern | refuse | `ListRequest.LabelPattern` | a glob label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.LabelRegex | refuse | `ListRequest.LabelRegex` | a regex label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — as for LabelPattern, listReadyWork publishes label_regex and the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.TitleContains | refuse | `ListRequest.TitleContains` | a title substring that differs from the title filter refuses | listIssues publishes one title parameter. A TitleContains alone, or equal to TitleSearch once both are lowered, is folded onto `title` before encoding (FoldTitleContains: both compile to the same LOWER(title) LIKE predicate on the strings.ToLower of the value), so only a pair that still differs reaches this row, and sending either half alone would answer a wider listing | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.DescContains | refuse | `ListRequest.DescContains` | a description substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.NotesContains | refuse | `ListRequest.NotesContains` | a notes substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.ExternalContains | refuse | `ListRequest.ExternalContains` | an external-ref substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.ExternalRef | refuse | `ListRequest.ExternalRef` | an exact external-ref filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.UpdatedAfter | refuse | `ListRequest.UpdatedAfter` | an updated-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — only the created bounds are published | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.UpdatedBefore | refuse | `ListRequest.UpdatedBefore` | an updated-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — only the created bounds are published | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.ClosedAfter | refuse | `ListRequest.ClosedAfter` | a closed-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.ClosedBefore | refuse | `ListRequest.ClosedBefore` | a closed-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.DeferAfter | refuse | `ListRequest.DeferAfter` | a defer-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.DeferBefore | refuse | `ListRequest.DeferBefore` | a defer-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.DueAfter | refuse | `ListRequest.DueAfter` | a due-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.DueBefore | refuse | `ListRequest.DueBefore` | a due-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.EmptyDesc | refuse | `ListRequest.EmptyDesc` | the empty-description predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.NoLabels | refuse | `ListRequest.NoLabels` | the no-labels predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.Priority | refuse | `ListRequest.Priority` | an exact priority filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes `priority`; the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.PriorityMin | refuse | `ListRequest.PriorityMin` | a minimum-priority bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.PriorityMax | refuse | `ListRequest.PriorityMax` | a maximum-priority bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.PinnedFlag | refuse | `ListRequest.PinnedFlag` | selecting the pinned-FLAG rows refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). The pinned STATUS is reachable through `status`; the flag is a separate predicate at any status and has no parameter | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.NoPinnedFlag | refuse | `ListRequest.NoPinnedFlag` | holding the unflagged predicate in place refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). It changes nothing on a default listing and NARROWS under `all` or a pinned/hooked status — which is precisely when dropping it would widen the answer | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.ExcludeTypes | refuse | `ListRequest.ExcludeTypes` | a type exclusion refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes exclude_type; the listing publishes only the four include_* toggles | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.NoParent | refuse | `ListRequest.NoParent` | the top-level-only predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.MolType | refuse | `ListRequest.MolType` | a molecule-type restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.WispType | refuse | `ListRequest.WispType` | a wisp-type restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). The listing now ADMITS the wisp plane — include_ephemeral landed upstream and the encoder sends it (L1 retired) — so this refusal is no longer redundant with an invisible plane: it is the only thing standing between a caller and a listing that returns every wisp type when they named one | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.DeferredFlag | refuse | `ListRequest.DeferredFlag` | the deferred predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | -| E-ListRequest.OverdueFlag | refuse | `ListRequest.OverdueFlag` | the overdue predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.ReadyFlag | refuse | `ListRequest.ReadyFlag` | switching a listing onto the ready set refuses | ReadyFlag is not a filter but a change of QUESTION: it selects the blocker-aware ready query, which reads a narrower filter vocabulary than a ListRequest can describe. Over the wire that question is listReadyWork, a different operation with a different parameter table. Routing it there would have to decide, silently, which of this request's fields survive the crossing — so v1 refuses and the retirement path is an explicit route in the store, not a re-encode here | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.IncludeComments | refuse | `ListRequest.IncludeComments` | hydrating every row's comment bodies refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). listIssues documents `comments` as always absent on its rows; a caller that needs an issue's comments reads getIssue with include_comments, one issue at a time | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | | E-ListRequest.Offset | refuse | `ListRequest.Offset` | a non-zero Offset refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). The portable page is the keyset position, which the pager walks (D8 row 1); the store-backed role refuses a non-zero Offset itself | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | @@ -307,57 +278,57 @@ The descendant walk reads the same IssueFilter as the exact-ids shape and reads | ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | |---|---|---|---|---|---|---| -| P-IssueFilter.ExcludeStatus | refuse | `IssueFilter.ExcludeStatus` | a status exclusion that is not the derived default refuses | listIssues publishes no exclude_status: the wire publishes the INTENT (`all`) and the server re-derives the exclusions from the same builder against its own authoritative vocabulary, which additionally heals L7's degraded client-side set for the walk. An exclusion list no intent reproduces is therefore unstatable, and dropping it would widen the listing to closed work | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.Pinned | refuse | `IssueFilter.Pinned` | a pinned predicate that is not the derived default refuses | the pinned default is the FOURTH derived member and has no parameter of its own: `--pinned` selects the flagged rows and `--no-pinned` holds the unflagged predicate in place under `--all`, and the wire can express neither. Its derived value is a function of the `all` intent alone — upstream #5333 stopped an every-status selector from forcing it, so `--status all` and `--all` are now one filter and one wire question — and a value that intent does not reproduce is a user's own predicate, which NARROWS, and that is precisely when dropping it would answer a different question | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExcludeStatus | refuse | `IssueFilter.ExcludeStatus` | a status exclusion that is not the derived default refuses | the walk sends the INTENT (`all`) and the server re-derives the exclusions from the same builder against its own authoritative vocabulary, which additionally heals L7's degraded client-side set for the walk. listIssues' exclude_status (behind `issues.list.filters`) narrows on top of those defaults, but the walk does not send it, so an exclusion list no intent reproduces is unstatable here, and dropping it would widen the listing to closed work | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Pinned | refuse | `IssueFilter.Pinned` | a pinned predicate that is not the derived default refuses | the pinned default is the FOURTH derived member and has no parameter of its own. `--pinned` selects the flagged rows and `--no-pinned` holds the unflagged predicate in place under `--all`; listIssues publishes both behind `issues.list.filters`, but the walk does not send them. Its derived value is a function of the `all` intent alone — upstream #5333 stopped an every-status selector from forcing it, so `--status all` and `--all` are now one filter and one wire question — and a value that intent does not reproduce is a user's own predicate, which NARROWS, and that is precisely when dropping it would answer a different question | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.IsTemplate | refuse | `IssueFilter.IsTemplate` | a template predicate that is not the derived default refuses | listIssues publishes include_templates, an intent, not an is_template predicate. `&false` is what the absent flag derives and sends nothing; nil is what `--include-templates` derives and sends the flag; `&true` selects templates ONLY, which no intent on this operation can say | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.ExcludeTypes | refuse | `IssueFilter.ExcludeTypes` | a type exclusion beyond the derived gate and infra members refuses | listIssues publishes include_gates and include_infra — intents — and no exclude_type. The gate member and the whole infra vocabulary are what their absent flags derive; a leftover member is a user's `--exclude-type`, and dropping it would return the type they asked to hide | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExcludeTypes | refuse | `IssueFilter.ExcludeTypes` | a type exclusion beyond the derived gate and infra members refuses | listIssues publishes include_gates and include_infra — intents — and, behind `issues.list.filters`, an exclude_type the walk does not send. The gate member and the whole infra vocabulary are what their absent flags derive; a leftover member is a user's `--exclude-type`, and dropping it would return the type they asked to hide | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.Ephemeral | refuse | `IssueFilter.Ephemeral` | an ephemeral predicate that is not the derived default refuses | BuildListFilter sets it only for an infra `--type`, from the same flag the walk already encodes; the wire publishes no ephemeral PREDICATE on the listing — include_ephemeral is an intent that ADMITS the plane, not a selector that restricts to it — so any other value is unstatable | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.SkipWisps | refuse | `IssueFilter.SkipWisps` | a wisp-merge opt-out that is not the derived default refuses | NARROWED by client wave ga-mijra, which closed the client-side gap this row used to record. The walk now encodes include_ephemeral — SkipWisps going false IS that intent, and it is read back off the member the way every other derived default is — so a wisp-inclusive descendant listing crosses instead of refusing, which is the shape gc's TierBoth reads take. What still refuses is a SkipWisps no set of intents reproduces: holding the plane out while include_infra or an infra `--type` admits it. BuildListFilter never builds that pair, so it is a hand-made filter rather than a flag combination, and dropping the opt-out would merge a plane the caller asked to skip | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.SortBy | degrade | `IssueFilter.SortBy` | the walk's sort key is DROPPED | findAllDescendants accumulates every level into a map keyed by id, so no per-level order reaches the answer: the walk's own assembly discards it before the caller sees it. This is a drop with nothing to drop, and the rendering order is the tree renderer's | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.SortDesc | degrade | `IssueFilter.SortDesc` | the walk's sort direction is DROPPED | the other half of the order findAllDescendants discards; see P-IssueFilter.SortBy | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.SkipLabels | degrade | `IssueFilter.SkipLabels` | the label-hydration opt-out is DROPPED | the listing publishes no hydration parameter, so the client cannot ask the server to skip the labels join. Dropping it hands the caller MORE data and never fewer rows, so it cannot be misread as a narrower or wider answer | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.SkipCounts | degrade | `IssueFilter.SkipCounts` | the cardinality-hydration opt-out is DROPPED | the same absence as SkipLabels, and here refusing would be actively wrong: every text rendering of `bd list` sets it, so a refusal would kill the walk's DEFAULT rendering — the one behavior D4 classifies the walk as served in order to keep | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.Priority | refuse | `IssueFilter.Priority` | a populated IssueFilter.Priority refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.LabelPattern | refuse | `IssueFilter.LabelPattern` | a populated IssueFilter.LabelPattern refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.LabelRegex | refuse | `IssueFilter.LabelRegex` | a populated IssueFilter.LabelRegex refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.TitleSearch | refuse | `IssueFilter.TitleSearch` | a populated IssueFilter.TitleSearch refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.IDPrefix | refuse | `IssueFilter.IDPrefix` | a populated IssueFilter.IDPrefix refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.SpecIDPrefix | refuse | `IssueFilter.SpecIDPrefix` | a populated IssueFilter.SpecIDPrefix refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.TitleContains | refuse | `IssueFilter.TitleContains` | a populated IssueFilter.TitleContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.DescriptionContains | refuse | `IssueFilter.DescriptionContains` | a populated IssueFilter.DescriptionContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.NotesContains | refuse | `IssueFilter.NotesContains` | a populated IssueFilter.NotesContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.ExternalRefContains | refuse | `IssueFilter.ExternalRefContains` | a populated IssueFilter.ExternalRefContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.ExternalRef | refuse | `IssueFilter.ExternalRef` | a populated IssueFilter.ExternalRef refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.UpdatedAfter | refuse | `IssueFilter.UpdatedAfter` | a populated IssueFilter.UpdatedAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.UpdatedBefore | refuse | `IssueFilter.UpdatedBefore` | a populated IssueFilter.UpdatedBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.ClosedAfter | refuse | `IssueFilter.ClosedAfter` | a populated IssueFilter.ClosedAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.ClosedBefore | refuse | `IssueFilter.ClosedBefore` | a populated IssueFilter.ClosedBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.StartedAfter | refuse | `IssueFilter.StartedAfter` | a populated IssueFilter.StartedAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.StartedBefore | refuse | `IssueFilter.StartedBefore` | a populated IssueFilter.StartedBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.AfterCreatedAt | refuse | `IssueFilter.AfterCreatedAt` | a populated IssueFilter.AfterCreatedAt refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.AfterID | refuse | `IssueFilter.AfterID` | a populated IssueFilter.AfterID refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.AfterPriority | refuse | `IssueFilter.AfterPriority` | a populated IssueFilter.AfterPriority refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.EmptyDescription | refuse | `IssueFilter.EmptyDescription` | a populated IssueFilter.EmptyDescription refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.NoAssignee | refuse | `IssueFilter.NoAssignee` | a populated IssueFilter.NoAssignee refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.NoLabels | refuse | `IssueFilter.NoLabels` | a populated IssueFilter.NoLabels refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.PriorityMin | refuse | `IssueFilter.PriorityMin` | a populated IssueFilter.PriorityMin refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.PriorityMax | refuse | `IssueFilter.PriorityMax` | a populated IssueFilter.PriorityMax refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.SourceRepo | refuse | `IssueFilter.SourceRepo` | a populated IssueFilter.SourceRepo refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.EphemeralTier | refuse | `IssueFilter.EphemeralTier` | a populated IssueFilter.EphemeralTier refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.IsBlocked | refuse | `IssueFilter.IsBlocked` | a populated IssueFilter.IsBlocked refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.NoParent | refuse | `IssueFilter.NoParent` | a populated IssueFilter.NoParent refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.MolType | refuse | `IssueFilter.MolType` | a populated IssueFilter.MolType refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.WispType | refuse | `IssueFilter.WispType` | a populated IssueFilter.WispType refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.Deferred | refuse | `IssueFilter.Deferred` | a populated IssueFilter.Deferred refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.DeferAfter | refuse | `IssueFilter.DeferAfter` | a populated IssueFilter.DeferAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.DeferBefore | refuse | `IssueFilter.DeferBefore` | a populated IssueFilter.DeferBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.DueAfter | refuse | `IssueFilter.DueAfter` | a populated IssueFilter.DueAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.DueBefore | refuse | `IssueFilter.DueBefore` | a populated IssueFilter.DueBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.Overdue | refuse | `IssueFilter.Overdue` | a populated IssueFilter.Overdue refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.IncludeDependencies | refuse | `IssueFilter.IncludeDependencies` | a populated IssueFilter.IncludeDependencies refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.NoIDShrink | refuse | `IssueFilter.NoIDShrink` | a populated IssueFilter.NoIDShrink refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.Offset | refuse | `IssueFilter.Offset` | a populated IssueFilter.Offset refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | -| P-IssueFilter.Lite | refuse | `IssueFilter.Lite` | a populated IssueFilter.Lite refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Priority | refuse | `IssueFilter.Priority` | a populated IssueFilter.Priority refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.LabelPattern | refuse | `IssueFilter.LabelPattern` | a populated IssueFilter.LabelPattern refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.LabelRegex | refuse | `IssueFilter.LabelRegex` | a populated IssueFilter.LabelRegex refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.TitleSearch | refuse | `IssueFilter.TitleSearch` | a populated IssueFilter.TitleSearch refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IDPrefix | refuse | `IssueFilter.IDPrefix` | a populated IssueFilter.IDPrefix refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SpecIDPrefix | refuse | `IssueFilter.SpecIDPrefix` | a populated IssueFilter.SpecIDPrefix refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.TitleContains | refuse | `IssueFilter.TitleContains` | a populated IssueFilter.TitleContains refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DescriptionContains | refuse | `IssueFilter.DescriptionContains` | a populated IssueFilter.DescriptionContains refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NotesContains | refuse | `IssueFilter.NotesContains` | a populated IssueFilter.NotesContains refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExternalRefContains | refuse | `IssueFilter.ExternalRefContains` | a populated IssueFilter.ExternalRefContains refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExternalRef | refuse | `IssueFilter.ExternalRef` | a populated IssueFilter.ExternalRef refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.UpdatedAfter | refuse | `IssueFilter.UpdatedAfter` | a populated IssueFilter.UpdatedAfter refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.UpdatedBefore | refuse | `IssueFilter.UpdatedBefore` | a populated IssueFilter.UpdatedBefore refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ClosedAfter | refuse | `IssueFilter.ClosedAfter` | a populated IssueFilter.ClosedAfter refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ClosedBefore | refuse | `IssueFilter.ClosedBefore` | a populated IssueFilter.ClosedBefore refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.StartedAfter | refuse | `IssueFilter.StartedAfter` | a populated IssueFilter.StartedAfter refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.StartedBefore | refuse | `IssueFilter.StartedBefore` | a populated IssueFilter.StartedBefore refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.AfterCreatedAt | refuse | `IssueFilter.AfterCreatedAt` | a populated IssueFilter.AfterCreatedAt refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.AfterID | refuse | `IssueFilter.AfterID` | a populated IssueFilter.AfterID refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.AfterPriority | refuse | `IssueFilter.AfterPriority` | a populated IssueFilter.AfterPriority refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.EmptyDescription | refuse | `IssueFilter.EmptyDescription` | a populated IssueFilter.EmptyDescription refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoAssignee | refuse | `IssueFilter.NoAssignee` | a populated IssueFilter.NoAssignee refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoLabels | refuse | `IssueFilter.NoLabels` | a populated IssueFilter.NoLabels refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.PriorityMin | refuse | `IssueFilter.PriorityMin` | a populated IssueFilter.PriorityMin refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.PriorityMax | refuse | `IssueFilter.PriorityMax` | a populated IssueFilter.PriorityMax refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SourceRepo | refuse | `IssueFilter.SourceRepo` | a populated IssueFilter.SourceRepo refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.EphemeralTier | refuse | `IssueFilter.EphemeralTier` | a populated IssueFilter.EphemeralTier refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IsBlocked | refuse | `IssueFilter.IsBlocked` | a populated IssueFilter.IsBlocked refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoParent | refuse | `IssueFilter.NoParent` | a populated IssueFilter.NoParent refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.MolType | refuse | `IssueFilter.MolType` | a populated IssueFilter.MolType refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.WispType | refuse | `IssueFilter.WispType` | a populated IssueFilter.WispType refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Deferred | refuse | `IssueFilter.Deferred` | a populated IssueFilter.Deferred refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DeferAfter | refuse | `IssueFilter.DeferAfter` | a populated IssueFilter.DeferAfter refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DeferBefore | refuse | `IssueFilter.DeferBefore` | a populated IssueFilter.DeferBefore refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DueAfter | refuse | `IssueFilter.DueAfter` | a populated IssueFilter.DueAfter refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DueBefore | refuse | `IssueFilter.DueBefore` | a populated IssueFilter.DueBefore refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Overdue | refuse | `IssueFilter.Overdue` | a populated IssueFilter.Overdue refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IncludeDependencies | refuse | `IssueFilter.IncludeDependencies` | a populated IssueFilter.IncludeDependencies refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoIDShrink | refuse | `IssueFilter.NoIDShrink` | a populated IssueFilter.NoIDShrink refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Offset | refuse | `IssueFilter.Offset` | a populated IssueFilter.Offset refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Lite | refuse | `IssueFilter.Lite` | a populated IssueFilter.Lite refuses on the descendant walk | the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | | P-IssueFilter.unrecognized | refuse | — | a descendant-walk filter whose intents this client cannot reproduce at all refuses | the inversion proves a candidate by RE-RUNNING workapi.BuildListFilter and comparing the derived members, so a candidate the builder itself rejects — an unknown status, a vocabulary the client's degraded copy does not carry (L7) — is never compared. When every candidate is rejected that way there is no member to blame, and refusing on the recognition is the only honest answer | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 L7 | TestTheParentWalkInvertsTheDerivedDefaults | ## Ready bridge / WorkFilter (D8) diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 514bbcfe0..8fa89ef0b 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -95,6 +95,7 @@ go_test( "large_apply_client_budget_test.go", "large_batch_apply_test.go", "list_comments_omitted_test.go", + "list_filters_test.go", "list_ids_test.go", "list_sort_test.go", "memories_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 34d140240..08c54ed1f 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -139,6 +139,60 @@ func (e CountDependencyEdgesParamsDirection) Valid() bool { } } +// Defines values for ListIssuesParamsMolType. +const ( + ListIssuesParamsMolTypePatrol ListIssuesParamsMolType = "patrol" + ListIssuesParamsMolTypeSwarm ListIssuesParamsMolType = "swarm" + ListIssuesParamsMolTypeWork ListIssuesParamsMolType = "work" +) + +// Valid indicates whether the value is a known member of the ListIssuesParamsMolType enum. +func (e ListIssuesParamsMolType) Valid() bool { + switch e { + case ListIssuesParamsMolTypePatrol: + return true + case ListIssuesParamsMolTypeSwarm: + return true + case ListIssuesParamsMolTypeWork: + return true + default: + return false + } +} + +// Defines values for ListIssuesParamsWispType. +const ( + ListIssuesParamsWispTypeError ListIssuesParamsWispType = "error" + ListIssuesParamsWispTypeEscalation ListIssuesParamsWispType = "escalation" + ListIssuesParamsWispTypeGcReport ListIssuesParamsWispType = "gc_report" + ListIssuesParamsWispTypeHeartbeat ListIssuesParamsWispType = "heartbeat" + ListIssuesParamsWispTypePatrol ListIssuesParamsWispType = "patrol" + ListIssuesParamsWispTypePing ListIssuesParamsWispType = "ping" + ListIssuesParamsWispTypeRecovery ListIssuesParamsWispType = "recovery" +) + +// Valid indicates whether the value is a known member of the ListIssuesParamsWispType enum. +func (e ListIssuesParamsWispType) Valid() bool { + switch e { + case ListIssuesParamsWispTypeError: + return true + case ListIssuesParamsWispTypeEscalation: + return true + case ListIssuesParamsWispTypeGcReport: + return true + case ListIssuesParamsWispTypeHeartbeat: + return true + case ListIssuesParamsWispTypePatrol: + return true + case ListIssuesParamsWispTypePing: + return true + case ListIssuesParamsWispTypeRecovery: + return true + default: + return false + } +} + // Defines values for ListIssuesParamsSort. const ( ListIssuesParamsSortCreated ListIssuesParamsSort = "created" @@ -1119,7 +1173,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.heartbeat`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`, `dependencies.blocking.isBlocked`, which announces that `GET /v0/beads/dependencies/blocking` accepts `is_blocked` and answers each item's stored, transitive `is_blocked` flag (see that parameter) rather than silently answering `unknown_parameter`, and `issues.update.claim`, which announces that `PATCH /v0/beads/issues/{id}` accepts `claim` (see `UpdateIssueRequest`) and resolves the claimed id on both planes, so a wisp is claimable there, rather than a `400` naming the member unknown, and `issues.create.gates`, which announces that `POST /v0/beads/issues` and `issues:batchApply` create items accept the gate members `await_type`, `await_id`, `timeout` and `waiters` (see `CreateIssueRequest`), and `issues.update.gates`, which announces that `PATCH /v0/beads/issues/{id}`'s `patch` and an `issues:batchApply` update item's `patch` accept `await_id` and `waiters` (see `IssuePatchBody`), each rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.heartbeat`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.update.persistence`, which announces that `PATCH /v0/beads/issues/{id}` accepts a `persistence` member in its `patch` (see `IssuePatchBody`), the plane move behind `bd promote`, rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`, `issues.list.noAssignee`, `issues.list.title` and `issues.list.ids`, which announce that `GET /v0/beads/issues` accepts `no_assignee`, `title` and the repeatable `id` respectively (`bd list --no-assignee`, `--title` and `--id`; see those parameters) rather than answering `unknown_parameter`, `dependencies.blocking.isBlocked`, which announces that `GET /v0/beads/dependencies/blocking` accepts `is_blocked` and answers each item's stored, transitive `is_blocked` flag (see that parameter) rather than silently answering `unknown_parameter`, and `issues.update.claim`, which announces that `PATCH /v0/beads/issues/{id}` accepts `claim` (see `UpdateIssueRequest`) and resolves the claimed id on both planes, so a wisp is claimable there, rather than a `400` naming the member unknown, and `issues.create.gates`, which announces that `POST /v0/beads/issues` and `issues:batchApply` create items accept the gate members `await_type`, `await_id`, `timeout` and `waiters` (see `CreateIssueRequest`), and `issues.update.gates`, which announces that `PATCH /v0/beads/issues/{id}`'s `patch` and an `issues:batchApply` update item's `patch` accept `await_id` and `waiters` (see `IssuePatchBody`), each rather than a `400` naming the member unknown, and `issues.list.filters`, which announces that `GET /v0/beads/issues` accepts the rest of `bd list`'s filter vocabulary (the updated, closed, defer and due bounds, the priority range, the text, external-ref, label-pattern, label-regex and spec matches, and the empty-description, no-labels, no-parent, exclude-type, exclude-status, pinned, no-pinned, deferred, overdue, mol-type and wisp-type filters; see those parameters) rather than answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2468,10 +2522,10 @@ type ListIssuesParams struct { // A `type` this workspace calls infra is the one NARROWING it lifts. Such a type on its own reads only that type's ephemeral rows, which is where creating one puts it; with `include_ephemeral` the type's durable rows (a promoted wisp, an issue retyped after it was created) are listed as well. `include_infra` alone keeps the narrowing. IncludeEphemeral *bool `form:"include_ephemeral,omitempty" json:"include_ephemeral,omitempty"` - // CreatedBefore Only issues created strictly before this instant (RFC 3339). + // CreatedBefore Only issues created strictly before this instant. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. CreatedBefore *time.Time `form:"created_before,omitempty" json:"created_before,omitempty"` - // CreatedAfter Only issues created strictly after this instant (RFC 3339). + // CreatedAfter Only issues created strictly after this instant. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. CreatedAfter *time.Time `form:"created_after,omitempty" json:"created_after,omitempty"` // MetadataField Top-level metadata equality filter as `key=value`, split on the first `=`. Repeatable. An invalid key is a 400. @@ -2498,6 +2552,156 @@ type ListIssuesParams struct { // Id Restrict to these canonical ids (repeat the parameter). AND-ed with every other filter, including the default exclusions, so a caller addressing rows regardless of status, type or tier also sends all=true, include_templates, include_gates, include_infra and include_ephemeral. At most 200 values; an id longer than storage holds, an empty one, one with leading or trailing whitespace, or one carrying a control character or a comma is a 400 `invalid_argument` on `param: "id"` — each value is matched exactly as sent, never trimmed or split. An id that names no row is simply not matched. Advertised by the behavior token `issues.list.ids`. Id *[]string `form:"id,omitempty" json:"id,omitempty"` + // TitleContains Case-insensitive substring match on the title: `bd list --title-contains`, spelled as `GET /v0/beads/issues:count` spells it. It overlaps `title` deliberately: both reach the role, which carries both fields. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + TitleContains *string `form:"title_contains,omitempty" json:"title_contains,omitempty"` + + // DescContains Case-insensitive substring match on the description: `bd list --desc-contains`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + DescContains *string `form:"desc_contains,omitempty" json:"desc_contains,omitempty"` + + // NotesContains Case-insensitive substring match on the notes: `bd list --notes-contains`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + NotesContains *string `form:"notes_contains,omitempty" json:"notes_contains,omitempty"` + + // ExternalContains Substring match on the external reference: `bd list --external-contains`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + ExternalContains *string `form:"external_contains,omitempty" json:"external_contains,omitempty"` + + // ExternalRef Exact match on the external reference: `bd list --external-ref`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + ExternalRef *string `form:"external_ref,omitempty" json:"external_ref,omitempty"` + + // Spec Only issues whose spec id starts with this prefix: `bd list --spec`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + Spec *string `form:"spec,omitempty" json:"spec,omitempty"` + + // LabelPattern Glob label filter: `bd list --label-pattern`, with `GET /v0/beads/ready`'s spelling. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + LabelPattern *string `form:"label_pattern,omitempty" json:"label_pattern,omitempty"` + + // LabelRegex Regular-expression label filter: `bd list --label-regex`, with `GET /v0/beads/ready`'s spelling. A pattern the database cannot compile is a 400 `invalid_value` on this parameter. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + LabelRegex *string `form:"label_regex,omitempty" json:"label_regex,omitempty"` + + // UpdatedAfter Only issues last updated strictly after this instant: `bd list --updated-after`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + UpdatedAfter *time.Time `form:"updated_after,omitempty" json:"updated_after,omitempty"` + + // UpdatedBefore Only issues last updated strictly before this instant: `bd list --updated-before`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + UpdatedBefore *time.Time `form:"updated_before,omitempty" json:"updated_before,omitempty"` + + // ClosedAfter Only issues closed strictly after this instant: `bd list --closed-after`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + ClosedAfter *time.Time `form:"closed_after,omitempty" json:"closed_after,omitempty"` + + // ClosedBefore Only issues closed strictly before this instant: `bd list --closed-before`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + ClosedBefore *time.Time `form:"closed_before,omitempty" json:"closed_before,omitempty"` + + // DeferAfter Only issues deferred until strictly after this instant: `bd list --defer-after`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + DeferAfter *time.Time `form:"defer_after,omitempty" json:"defer_after,omitempty"` + + // DeferBefore Only issues deferred until strictly before this instant: `bd list --defer-before`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + DeferBefore *time.Time `form:"defer_before,omitempty" json:"defer_before,omitempty"` + + // DueAfter Only issues due strictly after this instant: `bd list --due-after`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + DueAfter *time.Time `form:"due_after,omitempty" json:"due_after,omitempty"` + + // DueBefore Only issues due strictly before this instant: `bd list --due-before`. RFC 3339; compared strictly and at whole-second precision (the bound is truncated to the second before it is compared), and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose column is NULL never matches either bound. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + DueBefore *time.Time `form:"due_before,omitempty" json:"due_before,omitempty"` + + // Priority Exact priority: `bd list --priority`. Absent means unfiltered; `0` is a real priority. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + Priority *int `form:"priority,omitempty" json:"priority,omitempty"` + + // PriorityMin Lowest priority to list, inclusive: `bd list --priority-min`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + PriorityMin *int `form:"priority_min,omitempty" json:"priority_min,omitempty"` + + // PriorityMax Highest priority to list, inclusive: `bd list --priority-max`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + PriorityMax *int `form:"priority_max,omitempty" json:"priority_max,omitempty"` + + // EmptyDescription Only issues with an empty description: `bd list --empty-description`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + EmptyDescription *bool `form:"empty_description,omitempty" json:"empty_description,omitempty"` + + // NoLabels Only issues carrying no label: `bd list --no-labels`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + NoLabels *bool `form:"no_labels,omitempty" json:"no_labels,omitempty"` + + // NoParent Only top-level issues, those with no parent: `bd list --no-parent`. Sending it with `parent` is a 400 on `no_parent`, as on `GET /v0/beads/issues:count`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + NoParent *bool `form:"no_parent,omitempty" json:"no_parent,omitempty"` + + // ExcludeType Issue types to leave out (repeat the parameter; a value may also be comma-separated): `bd list --exclude-type`. Normalized by the role as the local listing normalizes it. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + ExcludeType *[]string `form:"exclude_type,omitempty" json:"exclude_type,omitempty"` + + // ExcludeStatus Statuses to leave out (repeat the parameter; a value may also be comma-separated), on top of what `status` or the default exclusions select; it never re-admits a row they hide. With `all` there are no default exclusions, so the listing keeps every status not named here (the `pinned` and `hooked` statuses and custom done or frozen statuses included unless named). Each name is validated against the workspace's statuses (built-in and custom), as `GET /v0/beads/issues:count` validates it: an unknown name is a 400 `invalid_value` on this parameter, never an exclusion that excludes nothing. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + ExcludeStatus *[]string `form:"exclude_status,omitempty" json:"exclude_status,omitempty"` + + // Pinned Only issues carrying the pinned FLAG, at any status: `bd list --pinned`. The pinned STATUS is `status`; the flag is a separate predicate. Sending it with `no_pinned` is a 400 on `no_pinned`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + Pinned *bool `form:"pinned,omitempty" json:"pinned,omitempty"` + + // NoPinned Only issues without the pinned flag, held in place where `all` or a pinned/hooked `status` would otherwise drop that predicate: `bd list --no-pinned`. Sending it with `pinned` is a 400 on `no_pinned`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + NoPinned *bool `form:"no_pinned,omitempty" json:"no_pinned,omitempty"` + + // Deferred Only deferred issues: `bd list --deferred`. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + Deferred *bool `form:"deferred,omitempty" json:"deferred,omitempty"` + + // Overdue Only overdue issues, those whose due time is before now: `bd list --overdue`. Now is the SERVER's clock when the query runs, not the client's. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + Overdue *bool `form:"overdue,omitempty" json:"overdue,omitempty"` + + // MolType Only molecules of this type: `bd list --mol-type`. Any other value, including an empty one, is a 400 naming the parameter. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + MolType *ListIssuesParamsMolType `form:"mol_type,omitempty" json:"mol_type,omitempty"` + + // WispType Only rows of this wisp type (a predicate on the column, not a plane selector): `bd list --wisp-type`. Any other value, including an empty one, is a 400 naming the parameter. + // + // Behind the `issues.list.filters` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses the request locally against a server that does not advertise the token, rather than paying for that round trip or answering a wider listing. + WispType *ListIssuesParamsWispType `form:"wisp_type,omitempty" json:"wisp_type,omitempty"` + // Sort Which of the two served total orders this page is in. `created` is `(created_at DESC, id ASC)`; `priority` is `(priority ASC, created_at DESC, id ASC)` — `bd list`'s flagless ordering, and the order `bd list --sort priority` produces. // // THE DEFAULT IS `created` AND WILL NOT CHANGE. It is what this operation served before this parameter existed, so moving it would alter which rows a truncated page contains for every client written against that behavior, with no error to notice it by. @@ -2529,6 +2733,12 @@ type ListIssuesParams struct { Brief *bool `form:"brief,omitempty" json:"brief,omitempty"` } +// ListIssuesParamsMolType defines parameters for ListIssues. +type ListIssuesParamsMolType string + +// ListIssuesParamsWispType defines parameters for ListIssues. +type ListIssuesParamsWispType string + // ListIssuesParamsSort defines parameters for ListIssues. type ListIssuesParamsSort string @@ -2594,7 +2804,7 @@ type ClaimNextIssueParams struct { // LabelPattern Glob matched against labels. LabelPattern *string `form:"label_pattern,omitempty" json:"label_pattern,omitempty"` - // LabelRegex Regular expression matched against labels. + // LabelRegex Regular expression matched against labels, compiled by the database in its own dialect. A pattern it cannot compile is a 400 `invalid_value` on this parameter. LabelRegex *string `form:"label_regex,omitempty" json:"label_regex,omitempty"` // Priority Exact priority (0 is a real value, not "unset"). @@ -2671,22 +2881,22 @@ type CountIssuesParams struct { // NotesContains Substring match on the notes. NotesContains *string `form:"notes_contains,omitempty" json:"notes_contains,omitempty"` - // CreatedAfter RFC 3339. Only issues created strictly after this instant. + // CreatedAfter RFC 3339. Only issues created strictly after this instant, compared at whole-second precision (the bound is truncated to the second before it is compared) and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. CreatedAfter *time.Time `form:"created_after,omitempty" json:"created_after,omitempty"` - // CreatedBefore RFC 3339. Only issues created strictly before this instant. + // CreatedBefore RFC 3339. Only issues created strictly before this instant, compared at whole-second precision (the bound is truncated to the second before it is compared) and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. CreatedBefore *time.Time `form:"created_before,omitempty" json:"created_before,omitempty"` - // UpdatedAfter RFC 3339. + // UpdatedAfter RFC 3339. Only issues last updated strictly after this instant, compared at whole-second precision (the bound is truncated to the second before it is compared) and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. UpdatedAfter *time.Time `form:"updated_after,omitempty" json:"updated_after,omitempty"` - // UpdatedBefore RFC 3339. + // UpdatedBefore RFC 3339. Only issues last updated strictly before this instant, compared at whole-second precision (the bound is truncated to the second before it is compared) and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. UpdatedBefore *time.Time `form:"updated_before,omitempty" json:"updated_before,omitempty"` - // ClosedAfter RFC 3339. Counting closed rows needs no `status` beside it: a bare count already includes them. + // ClosedAfter RFC 3339. Only issues closed strictly after this instant, compared at whole-second precision (the bound is truncated to the second before it is compared) and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose `closed_at` is NULL never matches either bound. Counting closed rows needs no `status` beside it: a bare count already includes them. ClosedAfter *time.Time `form:"closed_after,omitempty" json:"closed_after,omitempty"` - // ClosedBefore RFC 3339. + // ClosedBefore RFC 3339. Only issues closed strictly before this instant, compared at whole-second precision (the bound is truncated to the second before it is compared) and normalized to UTC, so a bound carrying an offset selects exactly what its `Z` twin does. A row whose `closed_at` is NULL never matches either bound. ClosedBefore *time.Time `form:"closed_before,omitempty" json:"closed_before,omitempty"` // EmptyDescription Only issues with no description. @@ -2739,7 +2949,7 @@ type CountIssuesParams struct { // Behind the `issues.count.scope` behavior token: an older server answers `unknown_parameter` for this name. ExcludeType *[]string `form:"exclude_type,omitempty" json:"exclude_type,omitempty"` - // ExcludeStatus Statuses to exclude from the count. Repeat the parameter, or pass a comma-separated list; entries are taken as written (no normalization) but — UNLIKE `status` and `exclude_type` above — each name IS validated against this workspace's vocabulary (built-in statuses plus this workspace's own custom ones): an unrecognized name is `invalid_argument` on this parameter with `reason: invalid_value`, rather than silently excluding nothing and overcounting. There is no listing counterpart to this parameter: `GET /v0/beads/issues` computes its default status exclusions internally from workspace configuration and exposes no equivalent knob, so this is a Count-only capability. + // ExcludeStatus Statuses to exclude from the count. Repeat the parameter, or pass a comma-separated list; entries are taken as written (no normalization) but — UNLIKE `status` and `exclude_type` above — each name IS validated against this workspace's vocabulary (built-in statuses plus this workspace's own custom ones): an unrecognized name is `invalid_argument` on this parameter with `reason: invalid_value`, rather than silently excluding nothing and overcounting. `GET /v0/beads/issues` takes the same parameter behind the `issues.list.filters` token and validates it by the same rule; there it narrows on top of the listing's default status exclusions, which a count does not apply. // // Behind the `issues.count.scope` behavior token: an older server answers `unknown_parameter` for this name. ExcludeStatus *[]string `form:"exclude_status,omitempty" json:"exclude_status,omitempty"` @@ -2822,7 +3032,7 @@ type ListReadyWorkParams struct { // LabelPattern Glob matched against labels. LabelPattern *string `form:"label_pattern,omitempty" json:"label_pattern,omitempty"` - // LabelRegex Regular expression matched against labels. + // LabelRegex Regular expression matched against labels, compiled by the database in its own dialect. A pattern it cannot compile is a 400 `invalid_value` on this parameter. LabelRegex *string `form:"label_regex,omitempty" json:"label_regex,omitempty"` // Priority Exact priority (0 is a real value, not "unset"). @@ -2890,7 +3100,7 @@ type CountReadyWorkParams struct { // LabelPattern Glob matched against labels. LabelPattern *string `form:"label_pattern,omitempty" json:"label_pattern,omitempty"` - // LabelRegex Regular expression matched against labels. + // LabelRegex Regular expression matched against labels, compiled by the database in its own dialect. A pattern it cannot compile is a 400 `invalid_value` on this parameter. LabelRegex *string `form:"label_regex,omitempty" json:"label_regex,omitempty"` // Priority Exact priority (0 is a real value, not "unset"). diff --git a/internal/httpapi/claim_next.go b/internal/httpapi/claim_next.go index 5059320bb..a8cd12f78 100644 --- a/internal/httpapi/claim_next.go +++ b/internal/httpapi/claim_next.go @@ -72,6 +72,12 @@ func (s *Server) handleClaimNext(w http.ResponseWriter, r *http.Request) { } result, err := claimer.ClaimNext(r.Context(), req) if err != nil { + // The claim's scan compiles label_regex in the database's own dialect, + // so a pattern it cannot compile is the caller's typo and wins nothing: + // the 400 on that parameter the ready listing gives for the same filter. + if s.refuseUncompilableRegex(w, r, err, req.Filter.LabelRegex) { + return + } s.failClaimNext(w, r, err) return } @@ -107,7 +113,9 @@ func (s *Server) claimNextActor(w http.ResponseWriter, r *http.Request) (string, // rather than an omission. There is no id to have missed, and a row another // agent took is simply not in the set this claim scanned — the role walks past // it. The whole refusal vocabulary beyond the edge is the role's ErrValidation, -// which reaches the wire as the 400 the document promises. +// which reaches the wire as the 400 the document promises, and the database's +// refusal of an uncompilable label_regex, which handleClaimNext attributes to +// that parameter before it gets here. // // The role's own validation is defensively unreachable: an empty actor is // refused at the edge, and the two fields it refuses beyond that — Limit and diff --git a/internal/httpapi/claim_next_test.go b/internal/httpapi/claim_next_test.go index 3fcd45083..d9eafff79 100644 --- a/internal/httpapi/claim_next_test.go +++ b/internal/httpapi/claim_next_test.go @@ -1,8 +1,10 @@ package httpapi import ( + "errors" "fmt" "net/http" + "net/url" "strings" "testing" @@ -236,6 +238,36 @@ func TestClaimNextMapsTheRolesValidationToA400(t *testing.T) { } } +// TestClaimNextAttributesAnUncompilableLabelRegex: the claim's scan compiles +// label_regex in the database's own dialect, so a pattern it cannot compile is +// the caller's typo — the 400 on label_regex the ready listing gives for the +// same filter — not a 500. The same text on a claim that sent no regex stays a +// server fault. +func TestClaimNextAttributesAnUncompilableLabelRegex(t *testing.T) { + engineErr := errors.New("claim next: Error 1105 (HY000): the given regular expression is invalid") + for _, tc := range []struct { + name, path string + status int + }{ + {"with a regex", claimNextPath + "?label_regex=" + url.QueryEscape("tech-("), http.StatusBadRequest}, + {"without a regex", claimNextPath, http.StatusInternalServerError}, + } { + t.Run(tc.name, func(t *testing.T) { + ts := newClaimNextServer(t, &roleReadyClaimer{err: engineErr}) + resp := ts.claimNext(t, tc.path, `{"actor":"poller"}`) + if resp.StatusCode != tc.status { + t.Fatalf("status = %d, want %d: %s", resp.StatusCode, tc.status, readAll(t, resp)) + } + if tc.status != http.StatusBadRequest { + return + } + if body := decodeBody(t, resp); body["code"] != string(CodeInvalidArgument) || body["param"] != "label_regex" || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("body = %v, want invalid_argument on label_regex with reason invalid_value", body) + } + }) + } +} + // TestClaimNextHasNoConflictAndNoNotFound states the operation's negative space // as a test. A row a racing agent took is not in the set this claim scanned, so // there is nothing to 409 about; there is no id in the path, so there is diff --git a/internal/httpapi/list_filters_test.go b/internal/httpapi/list_filters_test.go new file mode 100644 index 000000000..9665c633a --- /dev/null +++ b/internal/httpapi/list_filters_test.go @@ -0,0 +1,426 @@ +package httpapi + +import ( + "errors" + "net/http" + "net/url" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// listFiltersParams is the vocabulary issues.list.filters publishes on +// listIssues, in the document's order. It is a list here and not a read-back of +// listFilterParams alone because the parity test below compares the two: a name +// the handler reads and this list does not hold is an unannounced filter, and a +// name here the handler never reads is a document promise the server refuses. +var listFiltersParams = []string{ + "title_contains", "desc_contains", "notes_contains", "external_contains", "external_ref", + "spec", "label_pattern", "label_regex", + "updated_after", "updated_before", "closed_after", "closed_before", + "defer_after", "defer_before", "due_after", "due_before", + "priority", "priority_min", "priority_max", + "empty_description", "no_labels", "no_parent", "exclude_type", "exclude_status", + "pinned", "no_pinned", "deferred", "overdue", + "mol_type", "wisp_type", +} + +// TestListFilterParametersMatchTheDocument is mechanical in both directions: +// driving listFilterParams over an empty query yields exactly the names it +// reads (query.read; it is straight-line, so an empty query exercises every +// accessor), and each of those must be a listIssues parameter whose description +// cites the token, so an older-server client can tell why it was refused. +func TestListFilterParametersMatchTheDocument(t *testing.T) { + q := newQuery(url.Values{}) + var req issueops.ListRequest + listFilterParams(q, &req) + + var read []string + for name := range q.read { + read = append(read, name) + } + slices.Sort(read) + want := slices.Clone(listFiltersParams) + slices.Sort(want) + if !slices.Equal(read, want) { + t.Errorf("listFilterParams reads %v, want exactly %v", read, want) + } + if len(want) != 30 { + t.Errorf("issues.list.filters publishes %d parameters, want 30", len(want)) + } + + doc := loadSpec(t) + so := specOps(t, doc)["listIssues"] + for _, name := range listFiltersParams { + p := specParam(t, so, name) + desc, _ := p["description"].(string) + if !strings.Contains(desc, "`issues.list.filters`") { + t.Errorf("listIssues parameter %q does not cite issues.list.filters in its description", name) + } + } +} + +// TestListFilterEnumsMatchTheTypesVocabulary keeps mol_type's and wisp_type's +// documented enums equal to the server's accepted sets, which are read off the +// types package `bd list` validates against. +func TestListFilterEnumsMatchTheTypesVocabulary(t *testing.T) { + doc := loadSpec(t) + so := specOps(t, doc)["listIssues"] + for _, tc := range []struct { + param string + want []string + }{ + {"mol_type", molTypeNames()}, + {"wisp_type", wispTypeNames()}, + } { + schema, ok := specParam(t, so, tc.param)["schema"].(map[string]any) + if !ok { + t.Fatalf("%s has no schema", tc.param) + } + if got := toStrings(t, schema["enum"]); !slices.Equal(got, tc.want) { + t.Errorf("%s enum = %v, the server accepts %v", tc.param, got, tc.want) + } + } +} + +// TestListFilterParametersReachTheStorageFilter is the handler half of +// issues.list.filters: each parameter reaches the storage seam, through the +// Reader role, as the filter field the local `bd list` flag produces, and a +// listing that sends none narrows nothing. +func TestListFilterParametersReachTheStorageFilter(t *testing.T) { + const instant = "2026-03-04T05:06:07Z" + at := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + isAt := func(t *testing.T, name string, got *time.Time) { + t.Helper() + if got == nil || !got.Equal(at) { + t.Errorf("%s = %v, want %v", name, got, at) + } + } + isInt := func(t *testing.T, name string, got *int, want int) { + t.Helper() + if got == nil || *got != want { + t.Errorf("%s = %v, want %d", name, got, want) + } + } + for _, tc := range []struct { + query string + check func(t *testing.T, f types.IssueFilter) + }{ + {"", func(t *testing.T, f types.IssueFilter) { + if f.TitleContains != "" || f.DescriptionContains != "" || f.NotesContains != "" || + f.ExternalRefContains != "" || f.ExternalRef != nil || f.SpecIDPrefix != "" || + f.LabelPattern != "" || f.LabelRegex != "" || + f.UpdatedAfter != nil || f.UpdatedBefore != nil || f.ClosedAfter != nil || f.ClosedBefore != nil || + f.DeferAfter != nil || f.DeferBefore != nil || f.DueAfter != nil || f.DueBefore != nil || + f.Priority != nil || f.PriorityMin != nil || f.PriorityMax != nil || + f.EmptyDescription || f.NoLabels || f.NoParent || slices.Contains(f.ExcludeTypes, "epic") || + !slices.Equal(f.ExcludeStatus, []types.Status{types.StatusClosed, types.StatusPinned}) || + f.Deferred || f.Overdue || f.MolType != nil || f.WispType != nil { + t.Errorf("a listing with no filter parameter narrowed the filter: %+v", f) + } + if f.Pinned == nil || *f.Pinned { + t.Errorf("default Pinned = %v, want the role's false", f.Pinned) + } + }}, + {"title_contains=Routed", func(t *testing.T, f types.IssueFilter) { + if f.TitleContains != "Routed" || f.TitleSearch != "" { + t.Errorf("TitleContains = %q TitleSearch = %q, want Routed and empty", f.TitleContains, f.TitleSearch) + } + }}, + {"desc_contains=d", func(t *testing.T, f types.IssueFilter) { + if f.DescriptionContains != "d" { + t.Errorf("DescriptionContains = %q", f.DescriptionContains) + } + }}, + {"notes_contains=n", func(t *testing.T, f types.IssueFilter) { + if f.NotesContains != "n" { + t.Errorf("NotesContains = %q", f.NotesContains) + } + }}, + {"external_contains=gh-", func(t *testing.T, f types.IssueFilter) { + if f.ExternalRefContains != "gh-" { + t.Errorf("ExternalRefContains = %q", f.ExternalRefContains) + } + }}, + {"external_ref=gh-7", func(t *testing.T, f types.IssueFilter) { + if f.ExternalRef == nil || *f.ExternalRef != "gh-7" { + t.Errorf("ExternalRef = %v", f.ExternalRef) + } + }}, + {"spec=docs/", func(t *testing.T, f types.IssueFilter) { + if f.SpecIDPrefix != "docs/" { + t.Errorf("SpecIDPrefix = %q", f.SpecIDPrefix) + } + }}, + {"label_pattern=tech-*", func(t *testing.T, f types.IssueFilter) { + if f.LabelPattern != "tech-*" { + t.Errorf("LabelPattern = %q", f.LabelPattern) + } + }}, + {"label_regex=tech-.%2B", func(t *testing.T, f types.IssueFilter) { + if f.LabelRegex != "tech-.+" { + t.Errorf("LabelRegex = %q", f.LabelRegex) + } + }}, + {"updated_after=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "UpdatedAfter", f.UpdatedAfter) }}, + {"updated_before=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "UpdatedBefore", f.UpdatedBefore) }}, + {"closed_after=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "ClosedAfter", f.ClosedAfter) }}, + {"closed_before=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "ClosedBefore", f.ClosedBefore) }}, + {"defer_after=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "DeferAfter", f.DeferAfter) }}, + {"defer_before=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "DeferBefore", f.DeferBefore) }}, + {"due_after=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "DueAfter", f.DueAfter) }}, + {"due_before=" + instant, func(t *testing.T, f types.IssueFilter) { isAt(t, "DueBefore", f.DueBefore) }}, + {"priority=0", func(t *testing.T, f types.IssueFilter) { isInt(t, "Priority", f.Priority, 0) }}, + {"priority_min=1", func(t *testing.T, f types.IssueFilter) { isInt(t, "PriorityMin", f.PriorityMin, 1) }}, + {"priority_max=3", func(t *testing.T, f types.IssueFilter) { isInt(t, "PriorityMax", f.PriorityMax, 3) }}, + {"empty_description=true", func(t *testing.T, f types.IssueFilter) { + if !f.EmptyDescription { + t.Error("EmptyDescription not set") + } + }}, + {"no_labels=true", func(t *testing.T, f types.IssueFilter) { + if !f.NoLabels { + t.Error("NoLabels not set") + } + }}, + {"no_parent=true", func(t *testing.T, f types.IssueFilter) { + if !f.NoParent { + t.Error("NoParent not set") + } + }}, + {"exclude_type=epic&exclude_type=chore,bug", func(t *testing.T, f types.IssueFilter) { + // After the role's own default exclusions, which it prepends. + want := []types.IssueType{"epic", "chore", "bug"} + if n := len(f.ExcludeTypes); n < len(want) || !slices.Equal(f.ExcludeTypes[n-len(want):], want) { + t.Errorf("ExcludeTypes = %v, want %v (repeated, and each value comma-split by the role)", f.ExcludeTypes, want) + } + }}, + {"exclude_status=blocked&exclude_status=deferred,in_progress", func(t *testing.T, f types.IssueFilter) { + // APPENDED to the role's default exclusions (closed, pinned), + // which it computes first; never in place of them. + want := []types.Status{types.StatusClosed, types.StatusPinned, "blocked", "deferred", "in_progress"} + if !slices.Equal(f.ExcludeStatus, want) { + t.Errorf("ExcludeStatus = %v, want %v", f.ExcludeStatus, want) + } + }}, + {"exclude_status=closed&all=true", func(t *testing.T, f types.IssueFilter) { + // Under `all` there are no defaults, so the caller's list is the + // whole exclusion: the active-set read. + if want := []types.Status{types.StatusClosed}; !slices.Equal(f.ExcludeStatus, want) { + t.Errorf("ExcludeStatus = %v, want %v", f.ExcludeStatus, want) + } + }}, + {"pinned=true", func(t *testing.T, f types.IssueFilter) { + if f.Pinned == nil || !*f.Pinned { + t.Errorf("Pinned = %v, want true", f.Pinned) + } + }}, + {"no_pinned=true&all=true", func(t *testing.T, f types.IssueFilter) { + // `all` alone drops the pinned predicate; no_pinned holds it. + if f.Pinned == nil || *f.Pinned { + t.Errorf("Pinned = %v, want false held under all", f.Pinned) + } + }}, + {"deferred=true", func(t *testing.T, f types.IssueFilter) { + if !f.Deferred { + t.Error("Deferred not set") + } + }}, + {"overdue=true", func(t *testing.T, f types.IssueFilter) { + if !f.Overdue { + t.Error("Overdue not set") + } + }}, + {"mol_type=swarm", func(t *testing.T, f types.IssueFilter) { + if f.MolType == nil || *f.MolType != types.MolTypeSwarm { + t.Errorf("MolType = %v, want swarm", f.MolType) + } + }}, + {"wisp_type=gc_report", func(t *testing.T, f types.IssueFilter) { + if f.WispType == nil || *f.WispType != types.WispTypeGCReport { + t.Errorf("WispType = %v, want gc_report", f.WispType) + } + }}, + } { + t.Run(tc.query, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + path := "/v0/beads/issues" + if tc.query != "" { + path += "?" + tc.query + } + if resp := ts.get(t, path); resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d, want 200 (body %v)", path, resp.StatusCode, decodeBody(t, resp)) + } + filters := rec.searchFilters() + if len(filters) != 1 { + t.Fatalf("search calls = %d, want 1", len(filters)) + } + tc.check(t, filters[0]) + }) + } +} + +// TestListFilterTimeBoundsAreNormalizedToUTC: a bound carrying an offset +// reaches storage as the SAME instant in UTC. The storage layer formats a bound +// in its own zone and a zone-less timestamp column reads only the wall clock, +// so an un-normalized `+02:00` bound would compare two hours off. Every bound +// on the operation goes through the one decoder, the created bounds included. +func TestListFilterTimeBoundsAreNormalizedToUTC(t *testing.T) { + want := time.Date(2026, 3, 4, 3, 6, 7, 0, time.UTC) + for _, param := range []string{ + "created_after", "created_before", "updated_after", "updated_before", "closed_after", "closed_before", + "defer_after", "defer_before", "due_after", "due_before", + } { + t.Run(param, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + path := "/v0/beads/issues?" + param + "=" + url.QueryEscape("2026-03-04T05:06:07+02:00") + if resp := ts.get(t, path); resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d", path, resp.StatusCode) + } + f := rec.searchFilters()[0] + got := map[string]*time.Time{ + "created_after": f.CreatedAfter, "created_before": f.CreatedBefore, + "updated_after": f.UpdatedAfter, "updated_before": f.UpdatedBefore, + "closed_after": f.ClosedAfter, "closed_before": f.ClosedBefore, + "defer_after": f.DeferAfter, "defer_before": f.DeferBefore, + "due_after": f.DueAfter, "due_before": f.DueBefore, + }[param] + if got == nil || !got.Equal(want) { + t.Fatalf("%s = %v, want the instant %v", param, got, want) + } + if got.Location() != time.UTC { + t.Errorf("%s reached storage in zone %v, want UTC", param, got.Location()) + } + // The sub-second part survives the decode; truncation to the + // second is the storage layer's, on every route alike. + if got.Format(time.RFC3339) != "2026-03-04T03:06:07Z" { + t.Errorf("%s formats as %s, want 2026-03-04T03:06:07Z", param, got.Format(time.RFC3339)) + } + }) + } +} + +// TestListFilterMalformedValuesAreRefusedByName: a value the server cannot +// read is a 400 invalid_value naming the parameter — never ignored, which would +// widen the listing, and never unknown_parameter, which a client reads as skew. +func TestListFilterMalformedValuesAreRefusedByName(t *testing.T) { + for _, tc := range []struct{ param, value string }{ + {"updated_after", "yesterday"}, + {"closed_before", "2026-03-04"}, + {"due_after", "2026-03-04 05:06:07"}, + {"priority", "high"}, + {"priority_min", "1.5"}, + {"no_parent", "maybe"}, + {"overdue", "yes please"}, + {"mol_type", "squad"}, + {"wisp_type", "Heartbeat"}, + // Cross-vocabulary: each enum accepts its OWN list only, so a value + // valid for the other one is refused, not matched against nothing. + {"mol_type", "heartbeat"}, + {"wisp_type", "swarm"}, + // Present-but-empty is not absent: the role's empty pointer is a real + // `= ''` predicate, so reading it as absent would widen the listing. + {"wisp_type", ""}, + } { + t.Run(tc.param+"="+tc.value, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + resp := ts.get(t, "/v0/beads/issues?"+tc.param+"="+url.QueryEscape(tc.value)) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", resp.StatusCode) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) || body["param"] != tc.param || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("body = %v, want invalid_argument on %s with reason invalid_value", body, tc.param) + } + if n := len(rec.searchFilters()); n != 0 { + t.Errorf("a refused request reached storage %d time(s)", n) + } + }) + } +} + +// TestListFilterSingleValuedParametersRefuseARepeat: only exclude_type repeats; +// a second value of any other is refused rather than silently resolved to one. +func TestListFilterSingleValuedParametersRefuseARepeat(t *testing.T) { + ts, _ := newReadServer(t, Config{}) + resp := ts.get(t, "/v0/beads/issues?closed_after=2026-03-04T05:06:07Z&closed_after=2026-03-05T05:06:07Z") + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", resp.StatusCode) + } + if body := decodeBody(t, resp); body["param"] != "closed_after" { + t.Errorf("body = %v, want param closed_after", body) + } +} + +// TestListFilterPinnedPairIsRefused: `pinned` with `no_pinned` is the role's +// ErrValidation (workapi.BuildListFilter), the refusal `bd list --pinned +// --no-pinned` gives locally, surfaced as a 400 rather than PinnedFlag +// silently winning. +func TestListFilterPinnedPairIsRefused(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + resp := ts.get(t, "/v0/beads/issues?pinned=true&no_pinned=true") + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body %v)", resp.StatusCode, decodeBody(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != "no_pinned" || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("body = %v, want invalid_value on no_pinned", body) + } + if n := len(rec.searchFilters()); n != 0 { + t.Errorf("the contradictory pair reached storage %d time(s)", n) + } +} + +// TestListFilterParentPairIsRefused: `parent` with `no_parent` is refused the +// same way, in the sentence `bd list --parent X --no-parent` and the count +// surface refuse it with, rather than ANDed into the parent's edge-less +// dotted-id children. +func TestListFilterParentPairIsRefused(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + resp := ts.get(t, "/v0/beads/issues?parent=bd-1&no_parent=true") + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body %v)", resp.StatusCode, decodeBody(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != "no_parent" || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("body = %v, want invalid_value on no_parent", body) + } + if n := len(rec.searchFilters()); n != 0 { + t.Errorf("the contradictory pair reached storage %d time(s)", n) + } +} + +// TestAnInvalidLabelRegexIsABadRequestNotAServerFault: the database compiles +// label_regex in its own dialect and refuses a pattern it cannot compile with +// "the given regular expression is invalid". On the operations that carry the +// parameter that refusal is the caller's typo, a 400 on label_regex, not a 500 +// that pages as a server error. The same text on a request that sent NO regex +// stays a server fault: the attribution is to the parameter, not the words. +func TestAnInvalidLabelRegexIsABadRequestNotAServerFault(t *testing.T) { + engineErr := errors.New("search count issues: Error 1105 (HY000): the given regular expression is invalid") + for _, tc := range []struct { + name, path string + status int + }{ + {"listing", "/v0/beads/issues?label_regex=" + url.QueryEscape("tech-("), http.StatusBadRequest}, + {"ready", "/v0/beads/ready?label_regex=" + url.QueryEscape("tech-("), http.StatusBadRequest}, + {"listing without a regex", "/v0/beads/issues", http.StatusInternalServerError}, + } { + t.Run(tc.name, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + rec.err = engineErr + resp := ts.get(t, tc.path) + if resp.StatusCode != tc.status { + t.Fatalf("status = %d, want %d", resp.StatusCode, tc.status) + } + if tc.status != http.StatusBadRequest { + return + } + if body := decodeBody(t, resp); body["code"] != string(CodeInvalidArgument) || body["param"] != "label_regex" || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("body = %v, want invalid_argument on label_regex with reason invalid_value", body) + } + }) + } +} diff --git a/internal/httpapi/query.go b/internal/httpapi/query.go index 010596b08..363d3651f 100644 --- a/internal/httpapi/query.go +++ b/internal/httpapi/query.go @@ -172,7 +172,15 @@ func (q *query) boundedLimit(fallback, ceiling int) int { return *v } -// timestamp reads an optional RFC 3339 instant. +// timestamp reads an optional RFC 3339 instant, normalized to UTC. +// +// THE NORMALIZATION IS THE POINT, not tidiness. The storage layer binds a +// bound by formatting it (sqlbuild: RFC 3339 at whole seconds, in the value's +// own zone), and a database whose timestamp column has no zone — wall-clock +// UTC — reads that literal's wall clock and IGNORES its offset. A bound sent +// as `...+02:00` would then select two hours off the instant the caller named. +// In UTC the formatted wall clock is the instant on every backend, so an +// offset-bearing bound selects exactly the rows its `Z` twin does. func (q *query) timestamp(name string) *time.Time { raw := q.str(name) if raw == "" { @@ -183,6 +191,7 @@ func (q *query) timestamp(name string) *time.Time { q.invalid(name, fmt.Sprintf("%q is not an RFC 3339 timestamp", raw)) return nil } + t = t.UTC() return &t } diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 234d1fe18..d637aa243 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -85,7 +85,7 @@ func (s *Server) handleReady(w http.ResponseWriter, r *http.Request) { } page, err := rd.Ready(r.Context(), req) if err != nil { - s.failReadErr(w, r, err) + s.failRegexReadErr(w, r, err, req.LabelRegex) return } writeJSON(w, apigen.ReadyPage{ @@ -155,7 +155,7 @@ func (s *Server) handleCountReady(w http.ResponseWriter, r *http.Request) { } result, err := counter.CountReady(r.Context(), req) if err != nil { - s.failReadErr(w, r, err) + s.failRegexReadErr(w, r, err, req.LabelRegex) return } writeJSON(w, apigen.ReadyCount{Total: result.Total}) @@ -231,14 +231,11 @@ func countFilters(q *query) issueops.CountRequest { // into it, a plane-only count would also lose its templates and gates. IncludeEphemeral: q.boolean("include_ephemeral"), - // The four scope fields behind `issues.count.scope` (S8): ParentID and - // ExcludeTypes are read exactly as the listing reads them (q.str("parent"), - // q.list("exclude_type")), because the role documents its own ParentID and - // ExcludeTypes as spelling ListRequest's fields identically. NoParent and - // ExcludeStatus have no listing counterpart to mirror — NoParent is a new - // boolean switch and ExcludeStatus is a Count-only capability the listing - // never exposed — so both are read with this file's own conventions for - // their kind (q.boolean, q.list) rather than a listing precedent. + // The four scope fields behind `issues.count.scope` (S8), each read + // exactly as the listing reads its twin, because the role documents + // them as ListRequest's fields under the same names. NoParent and + // ExcludeStatus were Count-only when S8 added them; the listing took + // both later, behind `issues.list.filters`. ParentID: q.str("parent"), NoParent: q.boolean("no_parent"), ExcludeTypes: q.list("exclude_type"), @@ -392,6 +389,7 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { Limit: q.limit(), } + listFilterParams(q, &req) // THE ORDER AND THE CURSOR ARE ONE DECISION. Each served order is a keyset // contract — its own position shape and its own strictly-after predicate — @@ -439,7 +437,7 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { } page, err := rd.List(r.Context(), req) if err != nil { - s.failReadErr(w, r, err) + s.failRegexReadErr(w, r, err, req.LabelRegex) return } @@ -461,6 +459,106 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { writeJSON(w, body) } +// listFilterParams decodes the rest of `bd list`'s filter vocabulary, the +// members behind issues.list.filters (CapIssuesListFilters), into the request +// the Reader role reads. +// +// It is DECODE ONLY. Every member reaches the role as the value the caller +// sent, and workapi.BuildListFilter stays the one interpreter: how PinnedFlag +// meets AllFlag and Status, what a type exclusion normalizes to, what a bound +// compares against — all of that is the role's, on every route. The names are +// countIssues' wherever that operation already spells the member (countFilters +// above), so the same filter has one name on both operations. +// +// The two closed vocabularies are refused HERE with a 400 naming the +// parameter, as `bd list --mol-type` / `--wisp-type` refuse at the CLI: the +// role would match nothing for an unknown value, which a caller would read as +// "no such rows" rather than "no such type". +// +// It is a function of its own, beside countFilters, so a test can drive it +// over an empty query and read back the exact parameter set it asks for. +func listFilterParams(q *query, req *issueops.ListRequest) { + req.TitleContains = q.str("title_contains") + req.DescContains = q.str("desc_contains") + req.NotesContains = q.str("notes_contains") + req.ExternalContains = q.str("external_contains") + req.ExternalRef = q.str("external_ref") + req.SpecPrefix = q.str("spec") + req.LabelPattern = q.str("label_pattern") + req.LabelRegex = q.str("label_regex") + + req.UpdatedAfter = q.timestamp("updated_after") + req.UpdatedBefore = q.timestamp("updated_before") + req.ClosedAfter = q.timestamp("closed_after") + req.ClosedBefore = q.timestamp("closed_before") + req.DeferAfter = q.timestamp("defer_after") + req.DeferBefore = q.timestamp("defer_before") + req.DueAfter = q.timestamp("due_after") + req.DueBefore = q.timestamp("due_before") + + req.Priority = q.integer("priority") + req.PriorityMin = q.integer("priority_min") + req.PriorityMax = q.integer("priority_max") + + req.EmptyDesc = q.boolean("empty_description") + req.NoLabels = q.boolean("no_labels") + req.NoParent = q.boolean("no_parent") + req.ExcludeTypes = q.list("exclude_type") + // Validated by the role against the workspace's status vocabulary; an + // unknown name is a 400 on this parameter (invalidFilterParam), never an + // exclusion that silently excludes nothing. + req.ExcludeStatus = q.list("exclude_status") + + req.PinnedFlag = q.boolean("pinned") + req.NoPinnedFlag = q.boolean("no_pinned") + req.DeferredFlag = q.boolean("deferred") + // `now` is THIS server's clock: the role compares due_at against the time + // the query runs, which over http is the server's rather than the + // invoking process's. The document says so on the parameter. + req.OverdueFlag = q.boolean("overdue") + + if v, ok := closedEnum(q, "mol_type", molTypeNames()); ok { + mt := issueops.MolType(v) + req.MolType = &mt + } + if v, ok := closedEnum(q, "wisp_type", wispTypeNames()); ok { + wt := issueops.WispType(v) + req.WispType = &wt + } +} + +// closedEnum reads an optional closed-vocabulary parameter. PRESENT-BUT-EMPTY +// is refused with the unknown values rather than read as absent: an empty +// pointer member on the role is a real empty-string predicate, so a client that sent +// one and saw it treated as absent would get the wider listing. +func closedEnum(q *query, name string, allowed []string) (string, bool) { + if !q.has(name) { + return "", false + } + v := q.oneOf(name, "", allowed...) + if v == "" { + q.invalid(name, fmt.Sprintf("an empty value is not one of %s", strings.Join(allowed, ", "))) + return "", false + } + return v, true +} + +// molTypeNames and wispTypeNames are the closed vocabularies mol_type and +// wisp_type accept, read off the types package's own lists so the spec's enum +// (held equal by TestListFilterEnumsMatchTheTypesVocabulary) and this server +// cannot drift from what `bd list` validates. +func molTypeNames() []string { return enumNames(types.ValidMolTypes()) } + +func wispTypeNames() []string { return enumNames(types.ValidWispTypes()) } + +func enumNames[T ~string](vals []T) []string { + out := make([]string, len(vals)) + for i, v := range vals { + out[i] = string(v) + } + return out +} + // maxListIDs bounds listIssues' repeatable `id` filter. It is the spec's // `maxItems`, and TestListIDsBoundMatchesTheSpec holds the two together. const maxListIDs = 200 @@ -654,6 +752,41 @@ func (s *Server) failReadErr(w http.ResponseWriter, r *http.Request, err error) s.failErr(w, r, err) } +// invalidRegexMessage is the SQL engine's refusal of a pattern it cannot +// compile (Dolt, embedded and server alike: `Error 1105 (HY000): the given +// regular expression is invalid`). +const invalidRegexMessage = "regular expression is invalid" + +// failRegexReadErr is failReadErr for the operations that carry `label_regex`. +// +// The pattern is compiled by the DATABASE, in its own regex dialect, so the +// handler cannot validate it up front without refusing patterns the local +// route accepts. What it can do is attribute the database's compile refusal +// to the parameter that caused it: a request that SENT a regex and earned the +// engine's invalid-pattern error is the caller's typo, a 400 on label_regex, +// not a 500 that pages as a server fault. Any other failure, or this text on +// a request with no regex, keeps failReadErr's classification. +func (s *Server) failRegexReadErr(w http.ResponseWriter, r *http.Request, err error, labelRegex string) { + if s.refuseUncompilableRegex(w, r, err, labelRegex) { + return + } + s.failReadErr(w, r, err) +} + +// refuseUncompilableRegex answers err as that 400 on label_regex when it is +// the database's compile refusal of the pattern the request sent, and reports +// whether it did. It is split out for claimNext, which shares the ready +// listing's filter but not its failure mapping. +func (s *Server) refuseUncompilableRegex(w http.ResponseWriter, r *http.Request, err error, labelRegex string) bool { + if labelRegex == "" || !strings.Contains(err.Error(), invalidRegexMessage) { + return false + } + requestInfo(r.Context()).refuse("label_regex") + s.fail(w, r, InvalidArgument("label_regex", ReasonInvalidValue, + fmt.Sprintf("%q is not a regular expression the database can compile", labelRegex))) + return true +} + // invalidFilterParam maps a filter-construction failure back to the query // parameter that caused it. // @@ -683,6 +816,10 @@ func invalidFilterParam(err error) (string, bool) { return "exclude_status", true case strings.HasPrefix(msg, "--parent and --no-parent are mutually exclusive"): return "no_parent", true + // The listing role's refusal of the pinned pair (issues.list.filters): + // the same sentence `bd list --pinned --no-pinned` refuses with. + case strings.HasPrefix(msg, "--pinned and --no-pinned are mutually exclusive"): + return "no_pinned", true case strings.HasPrefix(msg, "invalid issue type "): return "type", true case strings.HasPrefix(msg, "invalid sort policy "): diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index 604fccadc..e4e2c464a 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -28,12 +28,18 @@ type recordingIssues struct { // index-aligned with search. queries []string items []*types.IssueWithCounts + // err, when set, is what both reads answer with after recording the + // filter: the storage-side failure a handler has to classify. + err error } func (f *recordingIssues) GetReadyWorkWithCounts(_ context.Context, filter types.WorkFilter) (domain.SearchCountsPage, error) { f.mu.Lock() f.ready = append(f.ready, filter) f.mu.Unlock() + if f.err != nil { + return domain.SearchCountsPage{}, f.err + } return domain.SearchCountsPage{Items: f.items}, nil } @@ -42,6 +48,9 @@ func (f *recordingIssues) SearchIssuesWithCounts(_ context.Context, query string f.search = append(f.search, filter) f.queries = append(f.queries, query) f.mu.Unlock() + if f.err != nil { + return domain.SearchCountsPage{}, f.err + } return domain.SearchCountsPage{Items: f.items}, nil } @@ -174,6 +183,9 @@ func TestABuilderRefusalIsTheDocumentedBadRequest(t *testing.T) { {"a has-metadata key on the count surface", "/v0/beads/issues:count?has_metadata_key=1bad", "has_metadata_key"}, {"an unknown exclude-status name on the count surface", "/v0/beads/issues:count?exclude_status=bogus", "exclude_status"}, {"parent and no_parent together on the count surface", "/v0/beads/issues:count?parent=bd-1&no_parent=true", "no_parent"}, + {"pinned and no_pinned together on the listing", "/v0/beads/issues?pinned=true&no_pinned=true", "no_pinned"}, + {"parent and no_parent together on the listing", "/v0/beads/issues?parent=bd-1&no_parent=true", "no_parent"}, + {"an unknown exclude-status name on the listing", "/v0/beads/issues?exclude_status=bogus", "exclude_status"}, } { t.Run(tc.name, func(t *testing.T) { ts, _ := newReadServer(t, Config{}) diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index cfcb3aaf2..c4a101cd9 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -157,6 +157,26 @@ const ( CapIssuesListTitle = "issues.list.title" ) +// CapIssuesListFilters is the behavior capability that advertises the rest of +// `bd list`'s filter vocabulary on GET /v0/beads/issues (reads.go +// listFilterParams, spec OpListIssues): the updated/closed/defer/due bounds, +// the priority range, the text and external-ref matches, the label +// pattern/regex, the spec prefix, the empty-description, no-labels, no-parent, +// pinned-flag, deferred and overdue predicates, the type and status exclusions +// and the mol/wisp types. Each is carried as the issueops.ListRequest member +// `bd list` sets and applied by the same Reader role (workapi.BuildListFilter) +// the local route reads through; the handler builds no filter. +// +// ONE token for the set, unlike the per-parameter list tokens above: these +// members land in one build and no server will ever carry a subset, so a split +// would mint about twenty tokens with no cutover distinction between them. +// +// The client refuses a request carrying any of them locally against a server +// that does not advertise this token, with ErrUnsupported naming it: an older +// server answers the parameter with `unknown_parameter`, and dropping one would +// answer a WIDER listing than asked for. +const CapIssuesListFilters = "issues.list.filters" + // CapListIDs is the behavior capability that advertises the repeatable `id` // filter on listIssues (issueops.ListRequest.IDFilter; `bd list --id`). It // rides beside `issues.list` for the reason CapIssuesListSort does: the @@ -1103,7 +1123,7 @@ var behaviorCapabilities = []string{ CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapIssuesCloseAutoCloseMolecule, CapIssuesListSearch, CapIssuesListNoAssignee, CapIssuesListTitle, CapListIDs, CapIssuesUpdatePersistence, CapDependenciesBlockingIsBlocked, CapIssuesUpdateClaim, - CapIssuesCreateGates, CapIssuesUpdateGates, + CapIssuesCreateGates, CapIssuesUpdateGates, CapIssuesListFilters, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 2ccb0f6d6..ba2105508 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -640,7 +640,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", "issues.close.autoCloseMolecule", "issues.list.search", "issues.list.noAssignee", "issues.list.title", "issues.list.ids", "issues.update.persistence", "dependencies.blocking.isBlocked", "issues.update.claim", - "issues.create.gates", "issues.update.gates") + "issues.create.gates", "issues.update.gates", "issues.list.filters") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 4596a1111..d4740b07a 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -482,7 +482,10 @@ paths: type: string - name: label_regex in: query - description: Regular expression matched against labels. + description: >- + Regular expression matched against labels, compiled by the + database in its own dialect. A pattern it cannot compile is a + 400 `invalid_value` on this parameter. schema: type: string - name: priority @@ -968,7 +971,10 @@ paths: type: string - name: label_regex in: query - description: Regular expression matched against labels. + description: >- + Regular expression matched against labels, compiled by the + database in its own dialect. A pattern it cannot compile is a + 400 `invalid_value` on this parameter. schema: type: string - name: priority @@ -1273,13 +1279,23 @@ paths: default: false - name: created_before in: query - description: Only issues created strictly before this instant (RFC 3339). + description: >- + Only issues created strictly before this instant. RFC 3339; + compared strictly and at whole-second precision (the bound is + truncated to the second before it is compared), and normalized + to UTC, so a bound carrying an offset selects exactly what its + `Z` twin does. schema: type: string format: date-time - name: created_after in: query - description: Only issues created strictly after this instant (RFC 3339). + description: >- + Only issues created strictly after this instant. RFC 3339; + compared strictly and at whole-second precision (the bound is + truncated to the second before it is compared), and normalized + to UTC, so a bound carrying an offset selects exactly what its + `Z` twin does. schema: type: string format: date-time @@ -1367,6 +1383,541 @@ paths: maxItems: 200 items: type: string + - name: title_contains + in: query + description: >- + Case-insensitive substring match on the title: `bd list + --title-contains`, spelled as `GET /v0/beads/issues:count` + spells it. It overlaps `title` deliberately: both reach the + role, which carries both fields. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: desc_contains + in: query + description: >- + Case-insensitive substring match on the description: `bd + list --desc-contains`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: notes_contains + in: query + description: >- + Case-insensitive substring match on the notes: `bd list + --notes-contains`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: external_contains + in: query + description: >- + Substring match on the external reference: `bd list + --external-contains`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: external_ref + in: query + description: >- + Exact match on the external reference: `bd list + --external-ref`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: spec + in: query + description: >- + Only issues whose spec id starts with this prefix: `bd list + --spec`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: label_pattern + in: query + description: >- + Glob label filter: `bd list --label-pattern`, with `GET + /v0/beads/ready`'s spelling. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: label_regex + in: query + description: >- + Regular-expression label filter: `bd list --label-regex`, + with `GET /v0/beads/ready`'s spelling. A pattern the database + cannot compile is a 400 `invalid_value` on this parameter. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + - name: updated_after + in: query + description: >- + Only issues last updated strictly after this instant: `bd + list --updated-after`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: updated_before + in: query + description: >- + Only issues last updated strictly before this instant: `bd + list --updated-before`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: closed_after + in: query + description: >- + Only issues closed strictly after this instant: `bd list + --closed-after`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: closed_before + in: query + description: >- + Only issues closed strictly before this instant: `bd list + --closed-before`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: defer_after + in: query + description: >- + Only issues deferred until strictly after this instant: `bd + list --defer-after`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: defer_before + in: query + description: >- + Only issues deferred until strictly before this instant: `bd + list --defer-before`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: due_after + in: query + description: >- + Only issues due strictly after this instant: `bd list + --due-after`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: due_before + in: query + description: >- + Only issues due strictly before this instant: `bd list + --due-before`. RFC 3339; compared strictly and at + whole-second precision (the bound is truncated to the second + before it is compared), and normalized to UTC, so a bound + carrying an offset selects exactly what its `Z` twin does. A + row whose column is NULL never matches either bound. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + format: date-time + - name: priority + in: query + description: >- + Exact priority: `bd list --priority`. Absent means + unfiltered; `0` is a real priority. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: integer + - name: priority_min + in: query + description: >- + Lowest priority to list, inclusive: `bd list + --priority-min`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: integer + - name: priority_max + in: query + description: >- + Highest priority to list, inclusive: `bd list + --priority-max`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: integer + - name: empty_description + in: query + description: >- + Only issues with an empty description: `bd list + --empty-description`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: no_labels + in: query + description: >- + Only issues carrying no label: `bd list --no-labels`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: no_parent + in: query + description: >- + Only top-level issues, those with no parent: `bd list + --no-parent`. Sending it with `parent` is a 400 on + `no_parent`, as on `GET /v0/beads/issues:count`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: exclude_type + in: query + description: >- + Issue types to leave out (repeat the parameter; a value may + also be comma-separated): `bd list --exclude-type`. + Normalized by the role as the local listing normalizes it. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + style: form + explode: true + schema: + type: array + items: + type: string + - name: exclude_status + in: query + description: >- + Statuses to leave out (repeat the parameter; a value may also be + comma-separated), on top of what `status` or the default + exclusions select; it never re-admits a row they hide. With + `all` there are no default exclusions, so the listing keeps + every status not named here (the `pinned` and `hooked` + statuses and custom done or frozen statuses included unless + named). + Each name is validated against the workspace's statuses (built-in + and custom), as `GET /v0/beads/issues:count` validates it: an + unknown name is a 400 `invalid_value` on this parameter, never an + exclusion that excludes nothing. + + + Behind the `issues.list.filters` behavior token: an older server + answers `unknown_parameter` for this name. A client with the + handshake's capabilities cached refuses the request locally + against a server that does not advertise the token, rather than + paying for that round trip or answering a wider listing. + style: form + explode: true + schema: + type: array + items: + type: string + - name: pinned + in: query + description: >- + Only issues carrying the pinned FLAG, at any status: `bd + list --pinned`. The pinned STATUS is `status`; the flag is a + separate predicate. Sending it with `no_pinned` is a 400 on + `no_pinned`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: no_pinned + in: query + description: >- + Only issues without the pinned flag, held in place where + `all` or a pinned/hooked `status` would otherwise drop that + predicate: `bd list --no-pinned`. Sending it with `pinned` + is a 400 on `no_pinned`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: deferred + in: query + description: >- + Only deferred issues: `bd list --deferred`. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: overdue + in: query + description: >- + Only overdue issues, those whose due time is before now: `bd + list --overdue`. Now is the SERVER's clock when the query + runs, not the client's. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: boolean + default: false + - name: mol_type + in: query + description: >- + Only molecules of this type: `bd list --mol-type`. Any other + value, including an empty one, is a 400 naming the + parameter. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + enum: + - swarm + - patrol + - work + - name: wisp_type + in: query + description: >- + Only rows of this wisp type (a predicate on the column, not + a plane selector): `bd list --wisp-type`. Any other value, + including an empty one, is a 400 naming the parameter. + + + Behind the `issues.list.filters` behavior token: an older + server answers `unknown_parameter` for this name. A client + with the handshake's capabilities cached refuses the request + locally against a server that does not advertise the token, + rather than paying for that round trip or answering a wider + listing. + schema: + type: string + enum: + - heartbeat + - ping + - patrol + - gc_report + - recovery + - error + - escalation - name: sort in: query description: >- @@ -2032,39 +2583,65 @@ paths: type: string - name: created_after in: query - description: RFC 3339. Only issues created strictly after this instant. + description: >- + RFC 3339. Only issues created strictly after this instant, + compared at whole-second precision (the bound is truncated to + the second before it is compared) and normalized to UTC, so a + bound carrying an offset selects exactly what its `Z` twin does. schema: type: string format: date-time - name: created_before in: query - description: RFC 3339. Only issues created strictly before this instant. + description: >- + RFC 3339. Only issues created strictly before this instant, + compared at whole-second precision (the bound is truncated to + the second before it is compared) and normalized to UTC, so a + bound carrying an offset selects exactly what its `Z` twin does. schema: type: string format: date-time - name: updated_after in: query - description: RFC 3339. + description: >- + RFC 3339. Only issues last updated strictly after this instant, + compared at whole-second precision (the bound is truncated to + the second before it is compared) and normalized to UTC, so a + bound carrying an offset selects exactly what its `Z` twin does. schema: type: string format: date-time - name: updated_before in: query - description: RFC 3339. + description: >- + RFC 3339. Only issues last updated strictly before this instant, + compared at whole-second precision (the bound is truncated to + the second before it is compared) and normalized to UTC, so a + bound carrying an offset selects exactly what its `Z` twin does. schema: type: string format: date-time - name: closed_after in: query description: >- - RFC 3339. Counting closed rows needs no `status` beside it: a bare - count already includes them. + RFC 3339. Only issues closed strictly after this instant, + compared at whole-second precision (the bound is truncated to + the second before it is compared) and normalized to UTC, so a + bound carrying an offset selects exactly what its `Z` twin does. + A row whose `closed_at` is NULL never matches either bound. + Counting closed rows needs no `status` beside it: a bare count + already includes them. schema: type: string format: date-time - name: closed_before in: query - description: RFC 3339. + description: >- + RFC 3339. Only issues closed strictly before this instant, + compared at whole-second precision (the bound is truncated to + the second before it is compared) and normalized to UTC, so a + bound carrying an offset selects exactly what its `Z` twin does. + A row whose `closed_at` is NULL never matches either bound. schema: type: string format: date-time @@ -2237,10 +2814,10 @@ paths: (built-in statuses plus this workspace's own custom ones): an unrecognized name is `invalid_argument` on this parameter with `reason: invalid_value`, rather than silently excluding nothing - and overcounting. There is no listing counterpart to this - parameter: `GET /v0/beads/issues` computes its default status - exclusions internally from workspace configuration and exposes no - equivalent knob, so this is a Count-only capability. + and overcounting. `GET /v0/beads/issues` takes the same + parameter behind the `issues.list.filters` token and validates + it by the same rule; there it narrows on top of the listing's + default status exclusions, which a count does not apply. Behind the `issues.count.scope` behavior token: an older server @@ -4971,7 +5548,10 @@ paths: type: string - name: label_regex in: query - description: Regular expression matched against labels. + description: >- + Regular expression matched against labels, compiled by the + database in its own dialect. A pattern it cannot compile is a + 400 `invalid_value` on this parameter. schema: type: string - name: priority @@ -8453,7 +9033,15 @@ components: announces that `PATCH /v0/beads/issues/{id}`'s `patch` and an `issues:batchApply` update item's `patch` accept `await_id` and `waiters` (see `IssuePatchBody`), each rather than a `400` naming - the member unknown. One behavior token is + the member unknown, and `issues.list.filters`, which announces + that `GET /v0/beads/issues` accepts the rest of `bd list`'s filter + vocabulary (the updated, closed, defer and due bounds, the + priority range, the text, external-ref, label-pattern, + label-regex and spec matches, and the empty-description, + no-labels, no-parent, exclude-type, exclude-status, pinned, + no-pinned, deferred, overdue, mol-type and wisp-type filters; + see those parameters) rather than answering + `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 1c04f4f95..1e5727ee0 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -5876,6 +5876,24 @@ "explode": true, "default": "false" }, + { + "schema": "param:listIssues", + "member": "query:closed_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:closed_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:created_after", @@ -5902,6 +5920,68 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:defer_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:defer_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:deferred", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:desc_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:due_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:due_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:empty_description", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, { "schema": "param:listIssues", "member": "query:exclude_label", @@ -5911,6 +5991,40 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:exclude_status", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:exclude_type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:external_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:external_ref", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:has_metadata_key", @@ -5982,6 +6096,22 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:label_pattern", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:label_regex", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:limit", @@ -6000,6 +6130,19 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:mol_type", + "type": "string", + "enum": [ + "patrol", + "swarm", + "work" + ], + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:no_assignee", @@ -6009,6 +6152,50 @@ "explode": true, "default": "false" }, + { + "schema": "param:listIssues", + "member": "query:no_labels", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:no_parent", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:no_pinned", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:notes_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:overdue", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, { "schema": "param:listIssues", "member": "query:parent", @@ -6017,6 +6204,39 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:pinned", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:priority", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:priority_max", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:priority_min", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:q", @@ -6038,6 +6258,14 @@ "explode": true, "default": "created" }, + { + "schema": "param:listIssues", + "member": "query:spec", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:status", @@ -6055,6 +6283,14 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:title_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:type", @@ -6063,6 +6299,41 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:updated_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:updated_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:wisp_type", + "type": "string", + "enum": [ + "error", + "escalation", + "gc_report", + "heartbeat", + "patrol", + "ping", + "recovery" + ], + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listMemories", "member": "query:search", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 20dace40a..3293d92ae 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -125,6 +125,7 @@ go_test( "insecure_credential_guard_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", + "list_filters_test.go", "list_scope_test.go", "list_search_test.go", "list_sort_test.go", diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel index d0de56d70..b6a908ad2 100644 --- a/internal/httpclient/encode/BUILD.bazel +++ b/internal/httpclient/encode/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "ledger.go", "ledger_fields.go", "ledger_render.go", + "list_filters.go", "list_fold.go", "parentwalk.go", "skew.go", @@ -26,6 +27,7 @@ go_test( srcs = [ "bijection_test.go", "ledger_render_test.go", + "list_filters_test.go", "list_fold_test.go", "oracle_test.go", "parentwalk_test.go", diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index e580d79b8..911a85242 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -315,9 +315,63 @@ func ListParams(req issueops.ListRequest) (url.Values, error) { b.boolean("brief", req.Brief) b.intPtr("limit", req.Limit) + b.listFilters(req) return b.done() } +// listFilters emits the rest of `bd list`'s filter vocabulary, the members +// listIssues carries behind issues.list.filters, in the document's order. The +// pre-dial gate (httpclient role_reader.go, listFiltersSet) asks the handshake +// for that token whenever one of these is set; this emitter assumes it did. +// +// The bounds travel as the instants the CLI resolved (relative and date-only +// values are read on the CLIENT's clock and zone, exactly as locally), and the +// server compares them at whole seconds as the local store does. +func (b *builder) listFilters(req issueops.ListRequest) { + b.str("title_contains", req.TitleContains) + b.str("desc_contains", req.DescContains) + b.str("notes_contains", req.NotesContains) + b.str("external_contains", req.ExternalContains) + b.str("external_ref", req.ExternalRef) + b.str("spec", req.SpecPrefix) + b.str("label_pattern", req.LabelPattern) + b.str("label_regex", req.LabelRegex) + + b.timestamp("updated_after", req.UpdatedAfter) + b.timestamp("updated_before", req.UpdatedBefore) + b.timestamp("closed_after", req.ClosedAfter) + b.timestamp("closed_before", req.ClosedBefore) + b.timestamp("defer_after", req.DeferAfter) + b.timestamp("defer_before", req.DeferBefore) + b.timestamp("due_after", req.DueAfter) + b.timestamp("due_before", req.DueBefore) + + b.intPtr("priority", req.Priority) + b.intPtr("priority_min", req.PriorityMin) + b.intPtr("priority_max", req.PriorityMax) + + b.boolean("empty_description", req.EmptyDesc) + b.boolean("no_labels", req.NoLabels) + b.boolean("no_parent", req.NoParent) + b.list("exclude_type", req.ExcludeTypes) + b.list("exclude_status", req.ExcludeStatus) + + b.boolean("pinned", req.PinnedFlag) + b.boolean("no_pinned", req.NoPinnedFlag) + b.boolean("deferred", req.DeferredFlag) + b.boolean("overdue", req.OverdueFlag) + + // A set pointer travels even when it points at "": the role reads that + // as a real `= ''` predicate, and the server refuses the empty value by + // name rather than reading it as absent — never a silently wider listing. + if req.MolType != nil { + b.v.Set("mol_type", string(*req.MolType)) + } + if req.WispType != nil { + b.v.Set("wisp_type", string(*req.WispType)) + } +} + // splitIDSet reads ListRequest.IDFilter's comma-separated set the way the role // does (workapi.BuildListFilter: split on commas, trim, drop empties and // repeats) and hands back one member per listIssues `id` value, which is the diff --git a/internal/httpclient/encode/ledger_fields.go b/internal/httpclient/encode/ledger_fields.go index 642d5afcb..a5aab0e86 100644 --- a/internal/httpclient/encode/ledger_fields.go +++ b/internal/httpclient/encode/ledger_fields.go @@ -103,48 +103,23 @@ func readyRequestRows() []Row { // longest population in the ledger, and deliberately so: `bd list` publishes // far more filters than the v0 listing does, and every one of them is a set // this client would widen if it dropped it. +// +// [Retired: the twenty-nine filter rows that stood here — the spec prefix, the +// label pattern/regex, the four text and two external-ref matches, the eight +// updated/closed/defer/due bounds, the priority triple, EmptyDesc, NoLabels, +// NoParent, ExcludeTypes, the pinned pair, the mol/wisp types and the +// deferred/overdue predicates. listIssues publishes every one of them behind +// the issues.list.filters behavior token, the encoder sends them (listTable +// maps each to its parameter), and httpReader.List refuses a request carrying +// one against a server that does not advertise the token, before dialing and +// naming the token, so none of them can be dropped. What is left below is the +// part of the vocabulary that is a different question (ReadyFlag), a different +// projection (IncludeComments) or a page shape (Offset).] func listRequestRows() []Row { listRefusal := func(field, what, why string) Row { return refusal("E-ListRequest."+field, tyListRequest, field, what, why, fieldSpec) } rows := []Row{ - listRefusal("SpecPrefix", "a spec-id prefix restriction refuses", noWireList), - listRefusal("LabelPattern", "a glob label filter refuses", - noWireList+" — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through"), - listRefusal("LabelRegex", "a regex label filter refuses", - noWireList+" — as for LabelPattern, listReadyWork publishes label_regex and the listing does not"), - listRefusal("TitleContains", "a title substring that differs from the title filter refuses", - "listIssues publishes one title parameter. A TitleContains alone, or equal to TitleSearch once both are lowered, is folded onto `title` before encoding (FoldTitleContains: both compile to the same LOWER(title) LIKE predicate on the strings.ToLower of the value), so only a pair that still differs reaches this row, and sending either half alone would answer a wider listing"), - listRefusal("DescContains", "a description substring filter refuses", noWireList), - listRefusal("NotesContains", "a notes substring filter refuses", noWireList), - listRefusal("ExternalContains", "an external-ref substring filter refuses", noWireList), - listRefusal("ExternalRef", "an exact external-ref filter refuses", noWireList), - listRefusal("UpdatedAfter", "an updated-after bound refuses", noWireList+" — only the created bounds are published"), - listRefusal("UpdatedBefore", "an updated-before bound refuses", noWireList+" — only the created bounds are published"), - listRefusal("ClosedAfter", "a closed-after bound refuses", noWireList), - listRefusal("ClosedBefore", "a closed-before bound refuses", noWireList), - listRefusal("DeferAfter", "a defer-after bound refuses", noWireList), - listRefusal("DeferBefore", "a defer-before bound refuses", noWireList), - listRefusal("DueAfter", "a due-after bound refuses", noWireList), - listRefusal("DueBefore", "a due-before bound refuses", noWireList), - listRefusal("EmptyDesc", "the empty-description predicate refuses", noWireList), - listRefusal("NoLabels", "the no-labels predicate refuses", noWireList), - listRefusal("Priority", "an exact priority filter refuses", - noWireList+" — listReadyWork publishes `priority`; the listing does not"), - listRefusal("PriorityMin", "a minimum-priority bound refuses", noWireList), - listRefusal("PriorityMax", "a maximum-priority bound refuses", noWireList), - listRefusal("PinnedFlag", "selecting the pinned-FLAG rows refuses", - noWireList+". The pinned STATUS is reachable through `status`; the flag is a separate predicate at any status and has no parameter"), - listRefusal("NoPinnedFlag", "holding the unflagged predicate in place refuses", - noWireList+". It changes nothing on a default listing and NARROWS under `all` or a pinned/hooked status — which is precisely when dropping it would widen the answer"), - listRefusal("ExcludeTypes", "a type exclusion refuses", - noWireList+" — listReadyWork publishes exclude_type; the listing publishes only the four include_* toggles"), - listRefusal("NoParent", "the top-level-only predicate refuses", noWireList), - listRefusal("MolType", "a molecule-type restriction refuses", noWireList), - listRefusal("WispType", "a wisp-type restriction refuses", - noWireList+". The listing now ADMITS the wisp plane — include_ephemeral landed upstream and the encoder sends it (L1 retired) — so this refusal is no longer redundant with an invisible plane: it is the only thing standing between a caller and a listing that returns every wisp type when they named one"), - listRefusal("DeferredFlag", "the deferred predicate refuses", noWireList), - listRefusal("OverdueFlag", "the overdue predicate refuses", noWireList), listRefusal("ReadyFlag", "switching a listing onto the ready set refuses", "ReadyFlag is not a filter but a change of QUESTION: it selects the blocker-aware ready query, which reads a narrower filter vocabulary than a ListRequest can describe. Over the wire that question is listReadyWork, a different operation with a different parameter table. "+ "Routing it there would have to decide, silently, which of this request's fields survive the crossing — so v1 refuses and the retirement path is an explicit route in the store, not a re-encode here"), @@ -298,8 +273,8 @@ func parentWalkRows() []Row { ID: "P-IssueFilter." + field, Kind: KindRefuse, Type: tyIssueFilter, Field: field, What: "a populated IssueFilter." + field + " refuses on the descendant walk", - Why: "listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. " + - "Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly", + Why: "the walk does not send it: its inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the walk encodes, and anything left over is a restriction the bridge would have to drop to answer at all — a dropped filter widens the set invisibly. " + + "Most such members do have a listIssues parameter now, behind `issues.list.filters`; the walk has not taken them on yet, and `--flat` or `--json` asks listIssues for the parent's children directly", SpecRow: parentWalkSpec, PinnedBy: pin, } } @@ -313,15 +288,15 @@ func parentWalkRows() []Row { rows := []Row{ derived("ExcludeStatus", "a status exclusion that is not the derived default refuses", - "listIssues publishes no exclude_status: the wire publishes the INTENT (`all`) and the server re-derives the exclusions from the same builder against its own authoritative vocabulary, which additionally heals L7's degraded client-side set for the walk. "+ - "An exclusion list no intent reproduces is therefore unstatable, and dropping it would widen the listing to closed work"), + "the walk sends the INTENT (`all`) and the server re-derives the exclusions from the same builder against its own authoritative vocabulary, which additionally heals L7's degraded client-side set for the walk. "+ + "listIssues' exclude_status (behind `issues.list.filters`) narrows on top of those defaults, but the walk does not send it, so an exclusion list no intent reproduces is unstatable here, and dropping it would widen the listing to closed work"), derived("Pinned", "a pinned predicate that is not the derived default refuses", - "the pinned default is the FOURTH derived member and has no parameter of its own: `--pinned` selects the flagged rows and `--no-pinned` holds the unflagged predicate in place under `--all`, and the wire can express neither. "+ + "the pinned default is the FOURTH derived member and has no parameter of its own. `--pinned` selects the flagged rows and `--no-pinned` holds the unflagged predicate in place under `--all`; listIssues publishes both behind `issues.list.filters`, but the walk does not send them. "+ "Its derived value is a function of the `all` intent alone — upstream #5333 stopped an every-status selector from forcing it, so `--status all` and `--all` are now one filter and one wire question — and a value that intent does not reproduce is a user's own predicate, which NARROWS, and that is precisely when dropping it would answer a different question"), derived("IsTemplate", "a template predicate that is not the derived default refuses", "listIssues publishes include_templates, an intent, not an is_template predicate. `&false` is what the absent flag derives and sends nothing; nil is what `--include-templates` derives and sends the flag; `&true` selects templates ONLY, which no intent on this operation can say"), derived("ExcludeTypes", "a type exclusion beyond the derived gate and infra members refuses", - "listIssues publishes include_gates and include_infra — intents — and no exclude_type. The gate member and the whole infra vocabulary are what their absent flags derive; a leftover member is a user's `--exclude-type`, and dropping it would return the type they asked to hide"), + "listIssues publishes include_gates and include_infra — intents — and, behind `issues.list.filters`, an exclude_type the walk does not send. The gate member and the whole infra vocabulary are what their absent flags derive; a leftover member is a user's `--exclude-type`, and dropping it would return the type they asked to hide"), derived("Ephemeral", "an ephemeral predicate that is not the derived default refuses", "BuildListFilter sets it only for an infra `--type`, from the same flag the walk already encodes; the wire publishes no ephemeral PREDICATE on the listing — include_ephemeral is an intent that ADMITS the plane, not a selector that restricts to it — so any other value is unstatable"), derived("SkipWisps", "a wisp-merge opt-out that is not the derived default refuses", @@ -356,7 +331,13 @@ func parentWalkRows() []Row { // parentWalkRefusals enumerates the IssueFilter members that refuse on the // descendant walk: everything BuildListFilter can carry that is neither a -// derived default nor a filter listIssues publishes. +// derived default nor a filter the walk encodes. +// +// Most of them have had a listIssues parameter since `issues.list.filters`. +// Moving one off this list is a change to the walk, and the walk must then +// check that token itself: it dials through the SearchIssues bridge, so the +// listing's pre-dial gate (role_reader.go refuseUnservedListFilters) never +// sees it. // // Written out rather than derived, for the reason issueFilterBridgeRefusals is: // a list computed from the struct would absorb a field added upstream, and the diff --git a/internal/httpclient/encode/list_filters.go b/internal/httpclient/encode/list_filters.go new file mode 100644 index 000000000..81f34755f --- /dev/null +++ b/internal/httpclient/encode/list_filters.go @@ -0,0 +1,42 @@ +package encode + +import ( + "reflect" + + "github.com/steveyegge/beads/issueops" +) + +// ListFiltersMembers are the issueops.ListRequest members listIssues carries +// behind the issues.list.filters behavior token: the rest of `bd list`'s filter +// vocabulary, published together in one server build. +// +// It is ONE list read three ways. listFilters (encode.go) emits each member; +// ListFiltersSet below asks whether a request carries any, which is the +// question the pre-dial gate (httpclient role_reader.go) puts to the handshake; +// and TestListFiltersMembersAreTheTokenedParameters holds the parameters these +// members map to in listTable equal to the listIssues parameters whose document +// description cites the token. A member added to one and not the others fails +// there rather than riding the wire ungated. +var ListFiltersMembers = []string{ + "TitleContains", "DescContains", "NotesContains", "ExternalContains", "ExternalRef", + "SpecPrefix", "LabelPattern", "LabelRegex", + "UpdatedAfter", "UpdatedBefore", "ClosedAfter", "ClosedBefore", + "DeferAfter", "DeferBefore", "DueAfter", "DueBefore", + "Priority", "PriorityMin", "PriorityMax", + "EmptyDesc", "NoLabels", "NoParent", "ExcludeTypes", "ExcludeStatus", + "PinnedFlag", "NoPinnedFlag", "DeferredFlag", "OverdueFlag", + "MolType", "WispType", +} + +// ListFiltersSet reports whether req carries any ListFiltersMembers member, +// by the encoder's own notion of populated: a non-nil pointer is set even to a +// zero value (priority 0 is a real priority), an empty slice is not. +func ListFiltersSet(req issueops.ListRequest) bool { + rv := reflect.ValueOf(req) + for _, name := range ListFiltersMembers { + if populated(rv.FieldByName(name)) { + return true + } + } + return false +} diff --git a/internal/httpclient/encode/list_filters_test.go b/internal/httpclient/encode/list_filters_test.go new file mode 100644 index 000000000..f2d771688 --- /dev/null +++ b/internal/httpclient/encode/list_filters_test.go @@ -0,0 +1,134 @@ +package encode + +import ( + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/spec" + "github.com/steveyegge/beads/issueops" + "gopkg.in/yaml.v3" +) + +// TestListFiltersMembersAreTheTokenedParameters holds three statements of the +// issues.list.filters vocabulary equal: ListFiltersMembers (what the pre-dial +// gate asks about), the parameters listTable maps those members to (what the +// encoder sends), and the listIssues parameters whose document description +// cites the token (what the server announces behind it). A member sent without +// the gate asking would earn an older server's 400; a gated member the server +// does not announce would refuse against a server that serves it. +func TestListFiltersMembersAreTheTokenedParameters(t *testing.T) { + table, ok := TableFor(OpListIssues, "") + if !ok { + t.Fatal("no listIssues table") + } + entry := map[string]FieldEntry{} + for _, e := range table.Fields { + entry[e.Name] = e + } + var sent []string + for _, member := range ListFiltersMembers { + if _, ok := reflect.TypeOf(issueops.ListRequest{}).FieldByName(member); !ok { + t.Errorf("ListFiltersMembers names %s, which issueops.ListRequest does not declare", member) + continue + } + e, ok := entry[member] + if !ok || e.Disposition != DispParam { + t.Errorf("listTable does not map %s to a parameter (entry %+v)", member, e) + continue + } + sent = append(sent, e.Param) + } + + var doc struct { + Paths map[string]map[string]struct { + OperationID string `yaml:"operationId"` + Parameters []struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + } `yaml:"parameters"` + } `yaml:"paths"` + } + if err := yaml.Unmarshal(spec.OpenAPIV0(), &doc); err != nil { + t.Fatalf("parse the wire contract: %v", err) + } + var announced []string + for _, methods := range doc.Paths { + for _, op := range methods { + if op.OperationID != string(OpListIssues) { + continue + } + for _, p := range op.Parameters { + if strings.Contains(p.Description, "`issues.list.filters`") { + announced = append(announced, p.Name) + } + } + } + } + slices.Sort(sent) + slices.Sort(announced) + if !slices.Equal(sent, announced) { + t.Errorf("the gated members encode to %v, the document announces %v behind issues.list.filters", sent, announced) + } +} + +// TestListFiltersSetSeesEachMember: the gate's question is the encoder's own +// notion of populated, member by member. +func TestListFiltersSetSeesEachMember(t *testing.T) { + if ListFiltersSet(issueops.ListRequest{}) { + t.Error("an empty request reports a filter member") + } + if ListFiltersSet(issueops.ListRequest{ExcludeTypes: []string{}, Status: "open", TitleSearch: "x", Limit: new(int)}) { + t.Error("a request with only ungated members (and an empty exclusion list) reports a filter member") + } + zero := 0 + empty := issueops.MolType("") + for name, req := range map[string]issueops.ListRequest{ + "Priority=0": {Priority: &zero}, + "MolType=&\"\"": {MolType: &empty}, + "ExcludeTypes": {ExcludeTypes: []string{"epic"}}, + "ExcludeStatus": {ExcludeStatus: []string{"closed"}}, + "OverdueFlag": {OverdueFlag: true}, + "DueBefore": {DueBefore: &time.Time{}}, + } { + if !ListFiltersSet(req) { + t.Errorf("%s: ListFiltersSet = false, want true", name) + } + } +} + +// TestListParamsSendsTheFilterBoundsAsTheResolvedInstant pins how the bounds +// cross: the instant the CLI resolved, in UTC, with its sub-second part — the +// truncation to whole seconds is the storage layer's on both routes, so the +// wire must not pre-truncate (which would move a bound) or keep an offset. +func TestListParamsSendsTheFilterBoundsAsTheResolvedInstant(t *testing.T) { + at := time.Date(2026, 3, 4, 7, 6, 7, 900_000_000, time.FixedZone("UTC+2", 2*60*60)) + zero := 0 + empty := issueops.WispType("") + v, err := ListParams(issueops.ListRequest{ + ClosedAfter: &at, DueBefore: &at, Priority: &zero, WispType: &empty, + ExcludeTypes: []string{"epic", "chore"}, ExcludeStatus: []string{"closed", "blocked"}, + }) + if err != nil { + t.Fatalf("ListParams: %v", err) + } + for _, param := range []string{"closed_after", "due_before"} { + if got := v.Get(param); got != "2026-03-04T05:06:07.9Z" { + t.Errorf("%s = %q, want 2026-03-04T05:06:07.9Z", param, got) + } + } + if got := v.Get("priority"); got != "0" { + t.Errorf("priority = %q, want 0 (a set pointer to priority 0 is a real filter)", got) + } + if got, ok := v["wisp_type"]; !ok || !slices.Equal(got, []string{""}) { + t.Errorf("wisp_type = %v (present %v), want the empty value sent for the server to refuse by name", got, ok) + } + if got := v["exclude_status"]; !slices.Equal(got, []string{"closed", "blocked"}) { + t.Errorf("exclude_status = %v, want one value per member", got) + } + if got := v["exclude_type"]; !slices.Equal(got, []string{"epic", "chore"}) { + t.Errorf("exclude_type = %v, want one value per member", got) + } +} diff --git a/internal/httpclient/encode/list_fold.go b/internal/httpclient/encode/list_fold.go index f64855991..c7cfa0138 100644 --- a/internal/httpclient/encode/list_fold.go +++ b/internal/httpclient/encode/list_fold.go @@ -21,9 +21,9 @@ import ( // former is the store's own lowering: EqualFold also matches pairs such as // "ſ" and "s", which lower to different patterns. // -// When both are set and differ the request is returned unchanged: the listing -// publishes one title parameter, and the encoder refuses the leftover -// TitleContains (E-ListRequest.TitleContains) rather than drop either half. +// When both are set and differ the request is returned unchanged: the leftover +// TitleContains rides `title_contains` beside `title`, behind +// issues.list.filters (ListFiltersMembers), so neither half is dropped. // // The request is taken and returned by value; the caller's copy is never // mutated. diff --git a/internal/httpclient/encode/list_fold_test.go b/internal/httpclient/encode/list_fold_test.go index aee159feb..f7fd9f1d4 100644 --- a/internal/httpclient/encode/list_fold_test.go +++ b/internal/httpclient/encode/list_fold_test.go @@ -1,7 +1,6 @@ package encode import ( - "errors" "reflect" "testing" @@ -10,15 +9,16 @@ import ( // `bd list --title-contains X` folds onto the listing's `title` parameter: the // store compiles TitleContains and TitleSearch to one predicate, so the fold -// is exact. A pair that differs other than in letter case still refuses, -// because the listing publishes one title parameter and dropping either half -// would widen the answer. +// is exact. A pair that differs other than in letter case is two predicates, +// so it is left unfolded and sends both `title` and `title_contains` (the +// latter behind issues.list.filters): dropping either half would widen the +// answer. func TestFoldTitleContainsRidesTheTitleParameter(t *testing.T) { for _, tc := range []struct { - name string - req issueops.ListRequest - wantTitle []string - refused bool + name string + req issueops.ListRequest + wantTitle []string + wantTitleContains []string }{ {name: "title-contains alone", req: issueops.ListRequest{TitleContains: "Routed"}, wantTitle: []string{"Routed"}}, {name: "equal pair sends one title", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "Routed"}, wantTitle: []string{"Routed"}}, @@ -27,32 +27,22 @@ func TestFoldTitleContainsRidesTheTitleParameter(t *testing.T) { {name: "pair differing only in case sends one title", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "rOUTED"}, wantTitle: []string{"Routed"}}, {name: "title alone is untouched", req: issueops.ListRequest{TitleSearch: "Routed"}, wantTitle: []string{"Routed"}}, {name: "neither sends no title", req: issueops.ListRequest{}}, - {name: "differing pair refuses", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "task"}, refused: true}, + {name: "differing pair sends both", req: issueops.ListRequest{TitleSearch: "Routed", TitleContains: "task"}, wantTitle: []string{"Routed"}, wantTitleContains: []string{"task"}}, // strings.EqualFold matches "ſ" (long s) with "s", but the store lowers // them to different patterns, so this pair is two predicates. - {name: "pair equal only under case folding refuses", req: issueops.ListRequest{TitleSearch: "ſtep", TitleContains: "step"}, refused: true}, + {name: "pair equal only under case folding sends both", req: issueops.ListRequest{TitleSearch: "ſtep", TitleContains: "step"}, wantTitle: []string{"ſtep"}, wantTitleContains: []string{"step"}}, } { t.Run(tc.name, func(t *testing.T) { folded := FoldTitleContains(tc.req) params, err := ListParams(folded) - if tc.refused { - var refused *RefusedError - if !errors.As(err, &refused) { - t.Fatalf("ListParams(%+v) error = %v, want a *RefusedError", folded, err) - } - if refused.Row.ID != "E-ListRequest.TitleContains" { - t.Fatalf("refused by row %q, want E-ListRequest.TitleContains", refused.Row.ID) - } - return - } if err != nil { t.Fatalf("ListParams(%+v): %v", folded, err) } if got := params["title"]; !reflect.DeepEqual(got, tc.wantTitle) { t.Errorf("title = %q, want %q", got, tc.wantTitle) } - if got, ok := params["title_contains"]; ok { - t.Errorf("listIssues carries no title_contains parameter, but the encoder sent %q", got) + if got := params["title_contains"]; !reflect.DeepEqual(got, tc.wantTitleContains) { + t.Errorf("title_contains = %q, want %q", got, tc.wantTitleContains) } }) } diff --git a/internal/httpclient/encode/parentwalk.go b/internal/httpclient/encode/parentwalk.go index f9677388e..32b00eba5 100644 --- a/internal/httpclient/encode/parentwalk.go +++ b/internal/httpclient/encode/parentwalk.go @@ -19,8 +19,12 @@ import ( // findAllDescendants), and workapi.BuildListFilter populates ExcludeStatus, // Pinned, IsTemplate, ExcludeTypes, Ephemeral and SkipWisps on EVERY default // listing. So the "ParentID-only shape" D4 originally named can never occur, and -// encoding those members verbatim is impossible anyway: listIssues publishes no -// exclude_status, no exclude_type and no is_template. +// encoding those members verbatim was impossible when D4 was written: listIssues +// published no exclude_status, no exclude_type and no is_template. It still +// publishes no is_template. `issues.list.filters` has since published the other +// two, but as the caller's own exclusions, which the server appends to the +// defaults it derives, so a derived list sent verbatim would still not say what +// the intents say. The walk sends intents and does not consult that token. // // THE RESOLUTION (escalation ga-2ieek, spec revision 6b428bf63). The wire // publishes INTENTS — `all`, `include_templates`, `include_gates`, diff --git a/internal/httpclient/encode/parentwalk_test.go b/internal/httpclient/encode/parentwalk_test.go index f51b81a12..21b030f14 100644 --- a/internal/httpclient/encode/parentwalk_test.go +++ b/internal/httpclient/encode/parentwalk_test.go @@ -221,7 +221,8 @@ func TestTheParentWalkRefusesAResidualFilter(t *testing.T) { }{ { // The canonical residual: --exclude-type appends past the derived - // gate and infra members, and listIssues publishes no exclude_type. + // gate and infra members, and the walk does not send listIssues' + // exclude_type. name: "--exclude-type", in: textListing(issueops.ListRequest{ExcludeTypes: []string{"chore"}}), row: "P-IssueFilter.ExcludeTypes", @@ -236,8 +237,9 @@ func TestTheParentWalkRefusesAResidualFilter(t *testing.T) { }, { // A residual with a documented parameter is still a residual when - // the parameter is not this operation's: listIssues publishes no - // priority. + // the walk does not send it: listIssues takes `priority` only + // behind issues.list.filters, and the walk encodes none of that + // token's members. name: "--priority", in: textListing(issueops.ListRequest{Priority: intPtr(1)}), row: "P-IssueFilter.Priority", diff --git a/internal/httpclient/encode/roundtrip_test.go b/internal/httpclient/encode/roundtrip_test.go index f03f89604..47804f9a4 100644 --- a/internal/httpclient/encode/roundtrip_test.go +++ b/internal/httpclient/encode/roundtrip_test.go @@ -137,6 +137,8 @@ func roundTripCases() []roundTripCase { // survives the crossing only when the caller already held it in UTC is an // encoder that works by luck. zoned := time.Date(2025, 12, 31, 23, 59, 59, 0, time.FixedZone("KST", 9*3600)) + molType := issueops.MolType("patrol") + wispType := issueops.WispType("gc_report") zero, five := 0, 5 assignee, parent := "agent-7", "bd-epic1" @@ -203,6 +205,40 @@ func roundTripCases() []roundTripCase { Brief: true, Limit: &five, + + // The issues.list.filters vocabulary, every member at once. The pinned + // pair and parent/no-parent are contradictions the ROLE refuses; the + // wire carries them as written, and that is all this gate asks. + TitleContains: "routed", + DescContains: "repro", + NotesContains: "handoff", + ExternalContains: "gh-", + ExternalRef: "gh-42", + SpecPrefix: "docs/", + LabelPattern: "tech-*", + LabelRegex: "tech-(debt|legacy)", + UpdatedAfter: &subSecond, + UpdatedBefore: &zoned, + ClosedAfter: &zoned, + ClosedBefore: &subSecond, + DeferAfter: &subSecond, + DeferBefore: &zoned, + DueAfter: &zoned, + DueBefore: &subSecond, + Priority: &zero, + PriorityMin: &zero, + PriorityMax: &five, + EmptyDesc: true, + NoLabels: true, + NoParent: true, + ExcludeTypes: []string{"epic", "chore,bug"}, + ExcludeStatus: []string{"blocked", "deferred,in_progress"}, + PinnedFlag: true, + NoPinnedFlag: true, + DeferredFlag: true, + OverdueFlag: true, + MolType: &molType, + WispType: &wispType, } workEverything := types.WorkFilter{ diff --git a/internal/httpclient/encode/skew.go b/internal/httpclient/encode/skew.go index a7799fd96..ad88e4e94 100644 --- a/internal/httpclient/encode/skew.go +++ b/internal/httpclient/encode/skew.go @@ -79,11 +79,18 @@ var paramFlag = map[string]string{ "closed_before": "closed-before", "created_after": "created-after", "created_before": "created-before", + "defer_after": "defer-after", + "defer_before": "defer-before", + "deferred": "deferred", "desc_contains": "desc-contains", + "due_after": "due-after", + "due_before": "due-before", "empty_description": "empty-description", "exclude_label": "exclude-label", "exclude_status": "exclude-status", "exclude_type": "exclude-type", + "external_contains": "external-contains", + "external_ref": "external-ref", "has_metadata_key": "has-metadata-key", "id": "id", "include_comments": "include-comments", @@ -99,16 +106,21 @@ var paramFlag = map[string]string{ "label_regex": "label-regex", "limit": "limit", "metadata_field": "metadata-field", + "mol_type": "mol-type", "no_assignee": "no-assignee", "no_labels": "no-labels", "no_parent": "no-parent", + "no_pinned": "no-pinned", "notes_contains": "notes-contains", + "overdue": "overdue", "parent": "parent", + "pinned": "pinned", "priority": "priority", "priority_max": "priority-max", "priority_min": "priority-min", "reverse": "reverse", "sort": "sort", + "spec": "spec", "status": "status", "title": "title", "title_contains": "title-contains", @@ -116,6 +128,7 @@ var paramFlag = map[string]string{ "unassigned": "unassigned", "updated_after": "updated-after", "updated_before": "updated-before", + "wisp_type": "wisp-type", } // paramNoFlag names the wire members this client puts on a request that no flag diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index a18ad5907..d9041aa7e 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -324,6 +324,9 @@ func listTable() Table { }}, Fields: []FieldEntry{ param("Status", "status"), + // Behind issues.list.filters with the rest of the filter + // vocabulary (encode.ListFiltersMembers). + paramTo("ExcludeStatus", "exclude_status", "ExcludeStatus"), paramTo("IssueType", "type", "IssueType"), param("Assignee", "assignee"), // TitleSearch, IDFilter and NoAssignee (S15b) are each behind their own @@ -334,7 +337,11 @@ func listTable() Table { // way the role splits it and sent as the repeatable `id`, one value // per member (bd-enterprise's contract, adopted unchanged). paramTo("TitleSearch", "title", "TitleSearch"), - refused("SpecPrefix", "E-ListRequest.SpecPrefix"), + // Every member below mapped as paramTo and named by the + // issues.list.filters gate (role_reader.go listFiltersSet) rides + // that one behavior token: the rest of `bd list`'s filter + // vocabulary, published together. + paramTo("SpecPrefix", "spec", "SpecPrefix"), paramTo("IDFilter", "id", "IDFilter"), // Behind issues.list.search: httpReader.List refuses a request // carrying it, before dialing, unless the handshake advertises the @@ -343,30 +350,30 @@ func listTable() Table { param("Labels", "label"), param("LabelsAny", "label_any"), param("ExcludeLabels", "exclude_label"), - refused("LabelPattern", "E-ListRequest.LabelPattern"), - refused("LabelRegex", "E-ListRequest.LabelRegex"), + paramTo("LabelPattern", "label_pattern", "LabelPattern"), + paramTo("LabelRegex", "label_regex", "LabelRegex"), // A TitleContains alone, or equal to TitleSearch up to letter // case, never reaches the encoder: httpReader.List folds it onto // `title` first (FoldTitleContains). What is left is a pair that - // differs. - refused("TitleContains", "E-ListRequest.TitleContains"), - refused("DescContains", "E-ListRequest.DescContains"), - refused("NotesContains", "E-ListRequest.NotesContains"), - refused("ExternalContains", "E-ListRequest.ExternalContains"), - refused("ExternalRef", "E-ListRequest.ExternalRef"), + // differs, sent as both `title` and `title_contains`. + paramTo("TitleContains", "title_contains", "TitleContains"), + paramTo("DescContains", "desc_contains", "DescContains"), + paramTo("NotesContains", "notes_contains", "NotesContains"), + paramTo("ExternalContains", "external_contains", "ExternalContains"), + paramTo("ExternalRef", "external_ref", "ExternalRef"), paramTo("CreatedBefore", "created_before", "CreatedBefore"), paramTo("CreatedAfter", "created_after", "CreatedAfter"), - refused("UpdatedAfter", "E-ListRequest.UpdatedAfter"), - refused("UpdatedBefore", "E-ListRequest.UpdatedBefore"), - refused("ClosedAfter", "E-ListRequest.ClosedAfter"), - refused("ClosedBefore", "E-ListRequest.ClosedBefore"), - refused("DeferAfter", "E-ListRequest.DeferAfter"), - refused("DeferBefore", "E-ListRequest.DeferBefore"), - refused("DueAfter", "E-ListRequest.DueAfter"), - refused("DueBefore", "E-ListRequest.DueBefore"), - refused("EmptyDesc", "E-ListRequest.EmptyDesc"), + paramTo("UpdatedAfter", "updated_after", "UpdatedAfter"), + paramTo("UpdatedBefore", "updated_before", "UpdatedBefore"), + paramTo("ClosedAfter", "closed_after", "ClosedAfter"), + paramTo("ClosedBefore", "closed_before", "ClosedBefore"), + paramTo("DeferAfter", "defer_after", "DeferAfter"), + paramTo("DeferBefore", "defer_before", "DeferBefore"), + paramTo("DueAfter", "due_after", "DueAfter"), + paramTo("DueBefore", "due_before", "DueBefore"), + paramTo("EmptyDesc", "empty_description", "EmptyDesc"), paramTo("NoAssignee", "no_assignee", "NoAssignee"), - refused("NoLabels", "E-ListRequest.NoLabels"), + paramTo("NoLabels", "no_labels", "NoLabels"), dropped("SkipLabels", "E-ListRequest.SkipLabels", "the wire hydrates labels either way; dropping the opt-out costs a join and hands the caller more data, never less"), dropped("SkipCounts", "E-ListRequest.SkipCounts", @@ -374,22 +381,22 @@ func listTable() Table { param("Brief", "brief"), refused("IncludeComments", "E-ListRequest.IncludeComments"), refused("IncludeAllTypes", "E-ListRequest.IncludeAllTypes"), - refused("Priority", "E-ListRequest.Priority"), - refused("PriorityMin", "E-ListRequest.PriorityMin"), - refused("PriorityMax", "E-ListRequest.PriorityMax"), - refused("PinnedFlag", "E-ListRequest.PinnedFlag"), - refused("NoPinnedFlag", "E-ListRequest.NoPinnedFlag"), + paramTo("Priority", "priority", "Priority"), + paramTo("PriorityMin", "priority_min", "PriorityMin"), + paramTo("PriorityMax", "priority_max", "PriorityMax"), + paramTo("PinnedFlag", "pinned", "PinnedFlag"), + paramTo("NoPinnedFlag", "no_pinned", "NoPinnedFlag"), param("IncludeTemplates", "include_templates"), param("IncludeGates", "include_gates"), param("IncludeInfra", "include_infra"), param("IncludeEphemeral", "include_ephemeral"), - refused("ExcludeTypes", "E-ListRequest.ExcludeTypes"), + paramTo("ExcludeTypes", "exclude_type", "ExcludeTypes"), paramTo("ParentID", "parent", "ParentID"), - refused("NoParent", "E-ListRequest.NoParent"), - refused("MolType", "E-ListRequest.MolType"), - refused("WispType", "E-ListRequest.WispType"), - refused("DeferredFlag", "E-ListRequest.DeferredFlag"), - refused("OverdueFlag", "E-ListRequest.OverdueFlag"), + paramTo("NoParent", "no_parent", "NoParent"), + paramTo("MolType", "mol_type", "MolType"), + paramTo("WispType", "wisp_type", "WispType"), + paramTo("DeferredFlag", "deferred", "DeferredFlag"), + paramTo("OverdueFlag", "overdue", "OverdueFlag"), paramTo("MetadataFields", "metadata_field", "MetadataFields"), param("HasMetadataKey", "has_metadata_key"), paramTo("AllFlag", "all", "AllFlag"), @@ -643,8 +650,8 @@ func searchExactIDsTable() Table { // because this shape is where the two bridges stop agreeing: the walk carries // `bd list`'s whole built filter, so six of its members are DERIVED defaults // the server re-derives (DispInverted) and eleven more are explicit user filters -// listIssues publishes parameters for. The exact-ids shape still refuses every -// one of them, which is why the enumeration cannot be shared. +// it sends under their listIssues parameters. The exact-ids shape still refuses +// every one of them, which is why the enumeration cannot be shared. func searchParentWalkTable() Table { const inversionWhy = "a derived default of workapi.BuildListFilter, re-derived server-side from the intent parameters the encoder sends in its place (D4 parent-walk inversion)" inverted := func(name, ledger string) FieldEntry { diff --git a/internal/httpclient/list_filters_test.go b/internal/httpclient/list_filters_test.go new file mode 100644 index 000000000..204e00755 --- /dev/null +++ b/internal/httpclient/list_filters_test.go @@ -0,0 +1,133 @@ +package httpclient + +import ( + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// listFilterRequest populates exactly one encode.ListFiltersMembers member of a +// ListRequest with a value that constrains something, by reflection over the +// member list itself so a member added there is swept here without a new row. +func listFilterRequest(t *testing.T, member string) issueops.ListRequest { + t.Helper() + var req issueops.ListRequest + f := reflect.ValueOf(&req).Elem().FieldByName(member) + if !f.IsValid() { + t.Fatalf("ListRequest has no member %s", member) + } + switch f.Kind() { + case reflect.String: + f.SetString("x") + case reflect.Bool: + f.SetBool(true) + case reflect.Slice: + f.Set(reflect.ValueOf([]string{"epic"})) + case reflect.Pointer: + switch f.Type().Elem() { + case reflect.TypeOf(time.Time{}): + at := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + f.Set(reflect.ValueOf(&at)) + case reflect.TypeOf(0): + zero := 0 // priority 0 is a real priority: a set pointer is set + f.Set(reflect.ValueOf(&zero)) + case reflect.TypeOf(issueops.MolType("")): + mt := issueops.MolType("swarm") + f.Set(reflect.ValueOf(&mt)) + case reflect.TypeOf(issueops.WispType("")): + wt := issueops.WispType("patrol") + f.Set(reflect.ValueOf(&wt)) + default: + t.Fatalf("member %s: no probe value for %s", member, f.Type()) + } + default: + t.Fatalf("member %s: no probe value for kind %s", member, f.Kind()) + } + if member == "TitleContains" { + // A lone TitleContains folds onto `title` (encode.FoldTitleContains) + // and asks only for issues.list.title; it rides title_contains only + // beside a TitleSearch that differs from it. + req.TitleSearch = "y" + } + return req +} + +// TestListFiltersRideListIssuesBehindTheToken is the client half of +// issues.list.filters: against a server that advertises it, each member +// reaches listIssues as its parameter (listTable's mapping), never dropped. +func TestListFiltersRideListIssuesBehindTheToken(t *testing.T) { + table, ok := encode.TableFor(encode.OpListIssues, "") + if !ok { + t.Fatal("no listIssues table") + } + paramOf := map[string]string{} + for _, e := range table.Fields { + paramOf[e.Name] = e.Param + } + served := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListTitle, wire.CapListFilters}} + for _, member := range encode.ListFiltersMembers { + t.Run(member, func(t *testing.T) { + w := newCountingWire(`{"items":[],"has_more":false}`) + limit := 10 + req := listFilterRequest(t, member) + req.Limit = &limit + if _, err := listRoleWithSnapshot(t, w, served).List(t.Context(), req); err != nil { + t.Fatalf("List: %v", err) + } + param := paramOf[member] + if param == "" { + t.Fatalf("listTable maps %s to no parameter", member) + } + if !w.query(t).Has(param) { + t.Errorf("%s did not reach listIssues as %s (query %v)", member, param, w.query(t)) + } + }) + } +} + +// TestListFiltersRefuseLocallyWhenTheServerLacksTheToken is the skew side, for +// every member: against a server that advertises every OTHER list token but +// not issues.list.filters (the deployed reference server's shape and newer), +// a request carrying one refuses BEFORE dialing with the typed capability +// error naming the token. Sending it would earn a 400 unknown_parameter; +// dropping it would answer a wider listing than asked for. +func TestListFiltersRefuseLocallyWhenTheServerLacksTheToken(t *testing.T) { + masked := &apigen.ContextResponse{Capabilities: []string{ + "issues.list", wire.CapListSort, wire.CapListSearch, wire.CapListNoAssignee, wire.CapListTitle, wire.CapListIDs, + }} + for _, member := range encode.ListFiltersMembers { + t.Run(member, func(t *testing.T) { + w := newCountingWire(`{"items":[],"has_more":false}`) + _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), listFilterRequest(t, member)) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("List error = %v, want *storage.ErrUnsupported", err) + } + if unsup.Capability != wire.CapListFilters || unsup.Op != "Reader.List" { + t.Errorf("refusal = {Op: %q, Capability: %q}, want {Reader.List, %s}", unsup.Op, unsup.Capability, wire.CapListFilters) + } + if !strings.Contains(err.Error(), wire.CapListFilters) { + t.Errorf("refusal text %q does not name %s", err, wire.CapListFilters) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.dialed)) + } + }) + } + + // A listing with none of the members is unaffected by the missing token, + // and an EMPTY exclusion list constrains nothing, so it asks for nothing. + w := newCountingWire(`{"items":[],"has_more":false}`) + limit := 10 + if _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), issueops.ListRequest{Limit: &limit, ExcludeTypes: []string{}}); err != nil { + t.Fatalf("List without any filter member against the same server: %v", err) + } +} diff --git a/internal/httpclient/list_scope_test.go b/internal/httpclient/list_scope_test.go index 87e775f40..225603488 100644 --- a/internal/httpclient/list_scope_test.go +++ b/internal/httpclient/list_scope_test.go @@ -103,10 +103,11 @@ func TestListScopeFiltersRefuseLocallyWhenTheServerLacksTheToken(t *testing.T) { } } -// TestListTitleContainsThatDiffersFromTitleRefusesBeforeDialing: the listing -// publishes one title parameter, so a `--title` and `--title-contains` that -// differ cannot both ride it. The request refuses, naming the member, without -// a round trip and without dropping either half. +// TestListTitleContainsThatDiffersFromTitleRefusesBeforeDialing: a `--title` +// and `--title-contains` that differ cannot both ride `title`, so the leftover +// TitleContains rides `title_contains`, behind issues.list.filters. Against a +// server that serves the title token but not that one, the request refuses, +// naming the token, without a round trip and without dropping either half. func TestListTitleContainsThatDiffersFromTitleRefusesBeforeDialing(t *testing.T) { served := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListTitle}} w := newCountingWire(`{"items":[],"has_more":false}`) @@ -115,8 +116,8 @@ func TestListTitleContainsThatDiffersFromTitleRefusesBeforeDialing(t *testing.T) if err == nil { t.Fatal("List with a differing --title/--title-contains pair succeeded; one half was dropped") } - if !strings.Contains(err.Error(), "TitleContains") { - t.Errorf("refusal %q does not name TitleContains", err) + if !strings.Contains(err.Error(), wire.CapListFilters) { + t.Errorf("refusal %q does not name %s", err, wire.CapListFilters) } if len(w.dialed) != 0 { t.Errorf("dialed %d times, want 0", len(w.dialed)) diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index b66bef781..12f3977f3 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -81,9 +81,12 @@ func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueop // tokenedListFilters are the ListRequest members listIssues carries only on a // server advertising a behavior token of their own, each with that token: -// `bd search`'s text (S26) and `bd list --title`, `--id` and `--no-assignee` -// (S15b). `--title-contains` has no entry: List folds it onto TitleSearch -// first (encode.FoldTitleContains), so it is gated by issues.list.title. +// `bd search`'s text (S26), `bd list --title`, `--id` and `--no-assignee` +// (S15b), and the rest of `bd list`'s filters behind issues.list.filters. +// `--title-contains` is folded onto TitleSearch first +// (encode.FoldTitleContains), so alone it is gated by issues.list.title; +// only one that differs from `--title` keeps TitleContains, which rides +// `title_contains` with the filters. // Order decides which token a request carrying several names; each refusal is // complete on its own, so any order is honest. var tokenedListFilters = []struct { @@ -98,6 +101,10 @@ var tokenedListFilters = []struct { {wire.CapListIDs, func(r issueops.ListRequest) bool { return strings.TrimSpace(strings.ReplaceAll(r.IDFilter, ",", "")) != "" }}, + // The rest of `bd list`'s filter vocabulary, one token for the set: the + // members are encode.ListFiltersMembers, the same list the encoder emits + // from, so a member cannot ride the wire without asking for the token. + {wire.CapListFilters, encode.ListFiltersSet}, } // refuseUnservedListFilters is the pre-dial gate for the tokened members. An diff --git a/internal/httpclient/roles_test.go b/internal/httpclient/roles_test.go index 1f1e8cdf8..200dca051 100644 --- a/internal/httpclient/roles_test.go +++ b/internal/httpclient/roles_test.go @@ -239,13 +239,13 @@ func TestReadRolesRefuseWhatTheWireCannotCarry(t *testing.T) { t.Fatalf("IssueReader(): %v", err) } - _, err = reader.List(ctx, issueops.ListRequest{SpecPrefix: "docs/"}) + _, err = reader.List(ctx, issueops.ListRequest{Offset: 1}) var inexpressible *InexpressibleError if !errors.As(err, &inexpressible) { - t.Fatalf("List with a SpecPrefix = %v, want *InexpressibleError", err) + t.Fatalf("List with an Offset = %v, want *InexpressibleError", err) } - if inexpressible.Refused.Row.Field != "SpecPrefix" { - t.Errorf("the refusal names %q, want SpecPrefix", inexpressible.Refused.Row.Field) + if inexpressible.Refused.Row.Field != "Offset" { + t.Errorf("the refusal names %q, want Offset", inexpressible.Refused.Row.Field) } if inexpressible.Refused.Row.SpecRow == "" || inexpressible.Refused.Row.Why == "" { t.Error("the refusal carries a ledger row with no reason or no citation") diff --git a/internal/httpclient/served_counter_test.go b/internal/httpclient/served_counter_test.go index 7f11a4296..a3931eff9 100644 --- a/internal/httpclient/served_counter_test.go +++ b/internal/httpclient/served_counter_test.go @@ -37,15 +37,14 @@ func servedCounterFixture(t *testing.T, e *servedEnv, prefix string) conformance CountHistory: e.countHistory, AddDependency: e.addDependency, // List is deliberately left nil (unlike the dolt leg's wiring in - // internal/storage/dolt/counter_contract_test.go). This client's OWN - // ListRequest encoder table (encode/table.go's listTable) has - // pre-existing, permanent refusals for NoParent and ExcludeTypes - // (E-ListRequest.NoParent, E-ListRequest.ExcludeTypes) — a divergence - // on the LISTING operation unrelated to S8 — so wiring List here would - // make RunCounterNoParentMatchesListCardinality and - // RunCounterExcludeTypesMatchesListCardinality FAIL rather than the - // requireCounterList skip this fixture's own documented design - // intends for a leg that cannot express the comparison. + // internal/storage/dolt/counter_contract_test.go). NoParent and + // ExcludeTypes now ride the listing behind issues.list.filters, but the + // cardinality cases list with IncludeAllTypes, which this client's + // ListRequest encoder still refuses (E-ListRequest.IncludeAllTypes), so + // wiring List here would make RunCounter*MatchesListCardinality FAIL + // rather than take the requireCounterList skip this fixture's own + // documented design intends for a leg that cannot express the + // comparison. } } diff --git a/internal/httpclient/served_list_test.go b/internal/httpclient/served_list_test.go index f364de85f..33448b0a0 100644 --- a/internal/httpclient/served_list_test.go +++ b/internal/httpclient/served_list_test.go @@ -1465,23 +1465,20 @@ func TestServedReaderDoesNotMutateTheCallerRequestOverTheWire(t *testing.T) { func TestServedReaderRefusesTheFiltersTheWireCannotCarry(t *testing.T) { ctx := t.Context() _, reader := servedReader(t) - cutoff := time.Now().UTC().Add(-time.Hour) - priority := 1 + // The filter members that stood here (SpecPrefix, PinnedFlag, PriorityMax, + // UpdatedAfter, a TitleContains that differs from TitleSearch, + // ExcludeTypes) ride listIssues behind issues.list.filters now; their + // served answers are the Reader contract's filter cases, and their refusal + // against a server without the token is list_filters_test.go's. What is + // left is the vocabulary the wire still cannot put. for _, test := range []struct { field string req issueops.ListRequest }{ - {"SpecPrefix", issueops.ListRequest{SpecPrefix: "docs/"}}, {"ReadyFlag", issueops.ListRequest{ReadyFlag: true}}, {"Offset", issueops.ListRequest{Offset: 1}}, - {"PinnedFlag", issueops.ListRequest{PinnedFlag: true}}, - {"PriorityMax", issueops.ListRequest{PriorityMax: &priority}}, - {"UpdatedAfter", issueops.ListRequest{UpdatedAfter: &cutoff}}, - // A TitleContains alone folds onto `title`; only a pair that differs - // is left with no parameter to ride. - {"TitleContains", issueops.ListRequest{TitleSearch: "something", TitleContains: "anything"}}, - {"ExcludeTypes", issueops.ListRequest{ExcludeTypes: []string{"chore"}}}, + {"IncludeComments", issueops.ListRequest{IncludeComments: true}}, } { page, err := reader.List(ctx, test.req) if err == nil { diff --git a/internal/httpclient/served_reader_test.go b/internal/httpclient/served_reader_test.go index 46dd2a396..62cfc8229 100644 --- a/internal/httpclient/served_reader_test.go +++ b/internal/httpclient/served_reader_test.go @@ -27,8 +27,9 @@ import ( // those cases parked on E-ListRequest.IDFilter. It now rides `id` behind // issues.list.ids, which this in-process server serves, and they run as // written. What still parks is a case whose SUBJECT (or one arm) is a member -// the wire refuses — Offset, ReadyFlag, PinnedFlag, ExcludeTypes, WispType, -// SkipCounts — and each names that member's row. The refusals THEMSELVES are +// the wire refuses — Offset, ReadyFlag, SkipCounts — and each names that +// member's row. PinnedFlag, ExcludeTypes and WispType retired from this list +// with issues.list.filters, which this in-process server also serves. The refusals THEMSELVES are // asserted in TestServedReaderRefusesTheFiltersTheWireCannotCarry, so the // parking never stands in for an unpinned behaviour. @@ -133,10 +134,9 @@ func TestServedReaderOffsetSkipsTheRowsBeforeThePage(t *testing.T) { conformance.RunReaderOffsetSkipsTheRowsBeforeThePage(t, t.Context(), servedReaderFixture(t, "rdr")) } +// The two pinned-flag arms ride `pinned` / `no_pinned` behind +// issues.list.filters, so the case runs as written. func TestServedReaderListDefaultExclusionsAndTheirOverrides(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.PinnedFlag", readParkBead, - "two arms set PinnedFlag/NoPinnedFlag, which listIssues publishes no parameters for (the id scope "+ - "every arm uses now rides `id`)") conformance.RunReaderListDefaultExclusionsAndTheirOverrides(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -214,8 +214,9 @@ func TestServedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder(t *testing. } // The id-scoped List cases that parked on E-ListRequest.IDFilter until S15b -// put `id` on listIssues (issues.list.ids). They run as written now; the two -// below them still park, because their SUBJECT is a member the wire refuses. +// put `id` on listIssues (issues.list.ids). They run as written now; the one +// below them that still parks does so because its SUBJECT is Offset, a member +// the wire refuses. func TestServedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge(t *testing.T) { conformance.RunReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge(t, t.Context(), servedReaderFixture(t, "rdr")) @@ -251,20 +252,26 @@ func TestServedReaderListMaxRowsBoundaryIsLimitPlusOffset(t *testing.T) { conformance.RunReaderListMaxRowsBoundaryIsLimitPlusOffset(t, t.Context(), servedReaderFixture(t, "rdr")) } -// TestServedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt -// names WISPTYPE for the same reason: the classification predicate IS the -// subject, and listIssues publishes no parameter for it, so no rescoping saves -// the case. The plane knob it composes against — IncludeEphemeral — is served, -// and its own half is asserted by -// TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope. +// TestServedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt runs +// as written: the classification predicate rides `wisp_type` behind +// issues.list.filters beside the served IncludeEphemeral plane knob. func TestServedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.WispType", readParkBead, - "the wisp classification predicate is the case's subject and listIssues publishes no parameter for "+ - "it; the client refuses it rather than answering the whole admitted plane, which is the reading "+ - "the case exists to rule out") conformance.RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t, t.Context(), servedReaderFixture(t, "rdr")) } +// The list filter families behind issues.list.filters, run through the +// client's encoder and the server's decode: the eight time bounds at their +// exact boundaries (whole-second, strict, NULL never matching, an offset bound +// equal to its UTC twin) and one included and one excluded row per remaining +// family. +func TestServedReaderListTimeBoundsAreStrictAndWholeSecond(t *testing.T) { + conformance.RunReaderListTimeBoundsAreStrictAndWholeSecond(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListFilterFamiliesNarrowToTheirRows(t *testing.T) { + conformance.RunReaderListFilterFamiliesNarrowToTheirRows(t, t.Context(), servedReaderFixture(t, "rdr")) +} + func TestServedReaderListStatusAcceptsACommaSeparatedORSet(t *testing.T) { conformance.RunReaderListStatusAcceptsACommaSeparatedORSet(t, t.Context(), servedReaderFixture(t, "rdr")) } @@ -382,10 +389,58 @@ func TestServedReaderGetPopulatesRowVersionForAGuardedWrite(t *testing.T) { } } +// The List leg sets ExcludeTypes, which rides `exclude_type` behind +// issues.list.filters, so the case runs as written. func TestServedReaderDoesNotMutateTheCallerRequest(t *testing.T) { - skipKnownDivergence(t, "E-ListRequest.ExcludeTypes", readParkBead, - "the List leg of the tripwire sets ExcludeTypes, which listIssues publishes no parameter for (its id "+ - "scope now rides `id`); the promise is exercised over this wire by "+ - "TestServedReaderDoesNotMutateTheCallerRequestOverTheWire") conformance.RunReaderDoesNotMutateTheCallerRequest(t, t.Context(), servedReaderFixture(t, "rdr")) } + +// TestServedReaderListExcludeStatusAcceptsAWorkspaceCustomStatus is the +// served twin of the count's custom-status pin: ListRequest.ExcludeStatus is +// validated against the SERVER workspace's own statuses (status.custom), so a +// custom status is accepted and applied over the wire, and a name in no +// vocabulary is still refused. A leg test rather than a conformance case +// because ReaderFixture has no way to configure a workspace custom status. +func TestServedReaderListExcludeStatusAcceptsAWorkspaceCustomStatus(t *testing.T) { + e := newServedEnv(t, "lcs") + ctx := t.Context() + if err := e.setConfig(ctx, "status.custom", "review,qa"); err != nil { + t.Fatalf("SetConfig(status.custom): %v", err) + } + const openID, reviewID = "lcs-open", "lcs-review" + for _, id := range []string{openID, reviewID} { + if err := e.createIssue(ctx, &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "tester"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + if err := e.reference.UpdateIssue(ctx, reviewID, map[string]interface{}{"status": types.Status("review")}, "tester"); err != nil { + t.Fatalf("UpdateIssue(%s) to review: %v", reviewID, err) + } + reader, err := e.subject.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + + scope := issueops.ListRequest{IDFilter: openID + "," + reviewID} + page, err := reader.List(ctx, scope) + if err != nil { + t.Fatalf("List(unfiltered): %v", err) + } + if got := pageIDs(page); len(got) != 2 { + t.Fatalf("List(unfiltered) = %v, want both rows (a custom status with no category is not a default exclusion)", got) + } + + scope.ExcludeStatus = []string{"review"} + page, err = reader.List(ctx, scope) + if err != nil { + t.Fatalf("List(ExcludeStatus=review): %v, want the custom status accepted", err) + } + if got := pageIDs(page); len(got) != 1 || got[0] != openID { + t.Fatalf("List(ExcludeStatus=review) = %v, want [%s]", got, openID) + } + + scope.ExcludeStatus = []string{"reveiw"} + if _, err := reader.List(ctx, scope); err == nil { + t.Fatal("List(ExcludeStatus=reveiw) answered instead of refusing a status in no vocabulary") + } +} diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 5eae5a69c..7aa468d49 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -183,6 +183,17 @@ const ( CapListIDs = "issues.list.ids" ) +// CapListFilters is the behavior capability announcing that listIssues honors +// the rest of `bd list`'s filter vocabulary — the updated/closed/defer/due +// bounds, the priority range, the text and external-ref matches, the label +// pattern/regex, the spec prefix, the type and status exclusions, the mol/wisp +// types and the empty-description, no-labels, no-parent, pinned, no-pinned, +// deferred and overdue predicates — spelled exactly as httpapi's +// CapIssuesListFilters. One token for the set: the members land in one server +// build. The Reader's List refuses a request carrying any of them LOCALLY, +// naming this token, when the server does not advertise it. +const CapListFilters = "issues.list.filters" + // CapBlockingIsBlocked is the behavior capability announcing that // listBlockingAnnotations honors `is_blocked` (issueops.BlockingRequest. // IncludeIsBlocked), spelled exactly as httpapi's @@ -453,7 +464,7 @@ var behaviorCapabilities = []string{ CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, CapCloseAutoCloseMolecule, CapListSearch, CapListNoAssignee, CapListTitle, CapListIDs, CapIssuesUpdatePersistence, CapBlockingIsBlocked, CapIssuesUpdateClaim, - CapIssuesCreateGates, CapIssuesUpdateGates, + CapIssuesCreateGates, CapIssuesUpdateGates, CapListFilters, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index b3144310f..fffa640ef 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -158,6 +158,9 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapListTitle != httpapi.CapIssuesListTitle { t.Errorf("CapListTitle = %q, server says %q", CapListTitle, httpapi.CapIssuesListTitle) } + if CapListFilters != httpapi.CapIssuesListFilters { + t.Errorf("CapListFilters = %q, server says %q", CapListFilters, httpapi.CapIssuesListFilters) + } if CapListIDs != httpapi.CapListIDs { t.Errorf("CapListIDs = %q, server says %q", CapListIDs, httpapi.CapListIDs) } @@ -228,12 +231,12 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, httpapi.CapIssuesCloseAutoCloseMolecule, httpapi.CapIssuesListSearch, httpapi.CapIssuesListNoAssignee, httpapi.CapIssuesListTitle, httpapi.CapListIDs, httpapi.CapIssuesUpdatePersistence, httpapi.CapDependenciesBlockingIsBlocked, httpapi.CapIssuesUpdateClaim, - httpapi.CapIssuesCreateGates, httpapi.CapIssuesUpdateGates, + httpapi.CapIssuesCreateGates, httpapi.CapIssuesUpdateGates, httpapi.CapIssuesListFilters, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit, "+ "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule, issues.list.search, "+ - "issues.list.noAssignee, issues.list.title, issues.list.ids, issues.update.persistence, dependencies.blocking.isBlocked, issues.update.claim, issues.create.gates and issues.update.gates tokens", behaviorCapabilities) + "issues.list.noAssignee, issues.list.title, issues.list.ids, issues.update.persistence, dependencies.blocking.isBlocked, issues.update.claim, issues.create.gates, issues.update.gates and issues.list.filters tokens", behaviorCapabilities) } } diff --git a/internal/storage/dolt/reader_contract_test.go b/internal/storage/dolt/reader_contract_test.go index 99a1ff884..00399186a 100644 --- a/internal/storage/dolt/reader_contract_test.go +++ b/internal/storage/dolt/reader_contract_test.go @@ -261,6 +261,18 @@ func TestReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t *testi conformance.RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t, ctx, fixture) } +func TestReaderListTimeBoundsAreStrictAndWholeSecond(t *testing.T) { + fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") + defer cleanup() + conformance.RunReaderListTimeBoundsAreStrictAndWholeSecond(t, ctx, fixture) +} + +func TestReaderListFilterFamiliesNarrowToTheirRows(t *testing.T) { + fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") + defer cleanup() + conformance.RunReaderListFilterFamiliesNarrowToTheirRows(t, ctx, fixture) +} + func TestReaderListBriefDropsTheFreeFormTextAndNothingElse(t *testing.T) { fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") defer cleanup() diff --git a/internal/storage/embeddeddolt/counter_contract_test.go b/internal/storage/embeddeddolt/counter_contract_test.go index b39570cb3..b70c1540c 100644 --- a/internal/storage/embeddeddolt/counter_contract_test.go +++ b/internal/storage/embeddeddolt/counter_contract_test.go @@ -3,6 +3,7 @@ package embeddeddolt_test import ( + "errors" "testing" "github.com/steveyegge/beads/backend/conformance" @@ -184,3 +185,54 @@ func TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra(t t.Fatalf("Count(exclude-status=review).Total = %d, want 1", result.Total) } } + +// TestReaderListExcludeStatusAcceptsAWorkspaceCustomStatus is the listing's +// twin of the test above: ListRequest.ExcludeStatus shares the count's +// validation (workapi.ParseExcludeStatus) and must see the workspace's own +// custom statuses, so a custom status is accepted and narrows the listing, +// and a name in no vocabulary is refused. A leg-level test for the same +// reason: ReaderFixture cannot configure a workspace custom status. +func TestReaderListExcludeStatusAcceptsAWorkspaceCustomStatus(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "lstcs") + ctx := t.Context() + + if err := te.store.SetConfig(ctx, "status.custom", "review,qa"); err != nil { + t.Fatalf("SetConfig(status.custom): %v", err) + } + const openID, reviewID = "lstcs-open", "lstcs-review" + for _, id := range []string{openID, reviewID} { + if err := te.store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + if err := te.store.UpdateIssue(ctx, reviewID, map[string]interface{}{ + "status": types.Status("review"), + }, "tester"); err != nil { + t.Fatalf("UpdateIssue(%s) to review: %v", reviewID, err) + } + reader, err := te.store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + + scope := publicops.ListRequest{IDFilter: openID + "," + reviewID, ExcludeStatus: []string{"review"}} + page, err := reader.List(ctx, scope) + if err != nil { + t.Fatalf("List(ExcludeStatus=review): %v, want the custom status accepted", err) + } + if len(page.Items) != 1 || page.Items[0].ID != openID { + ids := make([]string, 0, len(page.Items)) + for _, it := range page.Items { + ids = append(ids, it.ID) + } + t.Fatalf("List(ExcludeStatus=review) = %v, want [%s]", ids, openID) + } + + scope.ExcludeStatus = []string{"reveiw"} + if _, err := reader.List(ctx, scope); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("List(ExcludeStatus=reveiw) = %v, want ErrValidation", err) + } +} diff --git a/internal/storage/embeddeddolt/reader_contract_test.go b/internal/storage/embeddeddolt/reader_contract_test.go index 2bbdfbc9f..3e9286096 100644 --- a/internal/storage/embeddeddolt/reader_contract_test.go +++ b/internal/storage/embeddeddolt/reader_contract_test.go @@ -247,6 +247,18 @@ func TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt( conformance.RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t, ctx, newEmbeddedReaderFixture(t, "rdr")) } +func TestEmbeddedReaderListTimeBoundsAreStrictAndWholeSecond(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunReaderListTimeBoundsAreStrictAndWholeSecond(t, ctx, newEmbeddedReaderFixture(t, "rdr")) +} + +func TestEmbeddedReaderListFilterFamiliesNarrowToTheirRows(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunReaderListFilterFamiliesNarrowToTheirRows(t, ctx, newEmbeddedReaderFixture(t, "rdr")) +} + func TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() diff --git a/internal/storage/uow/reader_contract_test.go b/internal/storage/uow/reader_contract_test.go index cd38f36c4..54bab3aa0 100644 --- a/internal/storage/uow/reader_contract_test.go +++ b/internal/storage/uow/reader_contract_test.go @@ -167,6 +167,12 @@ func TestReaderContract(t *testing.T) { t.Run("ListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt", func(t *testing.T) { conformance.RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t, ctx, fixture) }) + t.Run("ListTimeBoundsAreStrictAndWholeSecond", func(t *testing.T) { + conformance.RunReaderListTimeBoundsAreStrictAndWholeSecond(t, ctx, fixture) + }) + t.Run("ListFilterFamiliesNarrowToTheirRows", func(t *testing.T) { + conformance.RunReaderListFilterFamiliesNarrowToTheirRows(t, ctx, fixture) + }) t.Run("ListBriefDropsTheFreeFormTextAndNothingElse", func(t *testing.T) { conformance.RunReaderListBriefDropsTheFreeFormTextAndNothingElse(t, ctx, fixture) }) diff --git a/internal/types/types.go b/internal/types/types.go index 45efe6938..5091abd86 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -931,6 +931,10 @@ func ValidMolTypeNames() string { return joinNamesWithOr(validMolTypes) } +// ValidMolTypes returns the accepted non-empty mol-type values, in canonical +// order, as a copy the caller may keep. +func ValidMolTypes() []MolType { return slices.Clone(validMolTypes) } + // WispType categorizes ephemeral wisps for TTL-based compaction (gt-9br) type WispType string @@ -970,6 +974,10 @@ func ValidWispTypeNames() string { return joinNamesWithOr(validWispTypes) } +// ValidWispTypes returns the accepted non-empty wisp-type values, in canonical +// order, as a copy the caller may keep. +func ValidWispTypes() []WispType { return slices.Clone(validWispTypes) } + // StorageClass is the create-selected marker for a record's history and // replication contract. Persistence plane transitions preserve the marker // except for normalization required by demotion or promotion. The effective diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 0931b468b..a39949cd8 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -144,20 +144,11 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt // rather than undercount, which is a worse failure mode than the // match-nothing treatment Status and IssueType accept above — so a typo // is ErrValidation instead. - for _, raw := range in.ExcludeStatus { - for _, s := range strings.Split(raw, ",") { - s = strings.TrimSpace(s) - if s == "" { - continue - } - status := types.Status(s) - if !status.IsValidWithCustom(cfg.CustomStatusNames()) { - return types.IssueFilter{}, fmt.Errorf("invalid exclude-status %q (valid: %s)%.0w", - s, ValidStatusList(cfg.CustomStatusNames()), issueops.ErrValidation) - } - filter.ExcludeStatus = append(filter.ExcludeStatus, status) - } + excluded, err := ParseExcludeStatus(in.ExcludeStatus, cfg.CustomStatusNames()) + if err != nil { + return types.IssueFilter{}, err } + filter.ExcludeStatus = append(filter.ExcludeStatus, excluded...) if in.IncludeInfra { applyCountIncludeInfra(&filter, in.IssueType, in.IncludeEphemeral, cfg) @@ -201,3 +192,28 @@ func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, include filter.Ephemeral = &ephemeral } } + +// ParseExcludeStatus reads a caller's status exclusion list — the count's and +// the listing's ExcludeStatus, one rule for both — into the statuses it names. +// Entries may be comma-separated and are trimmed; blanks are skipped. Each +// name IS validated against the workspace vocabulary (built-in statuses plus +// customStatusNames): a misspelled exclusion would silently exclude nothing +// and answer a WIDER set, so a typo is ErrValidation instead. +func ParseExcludeStatus(raw []string, customStatusNames []string) ([]types.Status, error) { + var out []types.Status + for _, entry := range raw { + for _, s := range strings.Split(entry, ",") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + status := types.Status(s) + if !status.IsValidWithCustom(customStatusNames) { + return nil, fmt.Errorf("invalid exclude-status %q (valid: %s)%.0w", + s, ValidStatusList(customStatusNames), issueops.ErrValidation) + } + out = append(out, status) + } + } + return out, nil +} diff --git a/internal/workapi/list.go b/internal/workapi/list.go index e5f9d7864..96a6750a6 100644 --- a/internal/workapi/list.go +++ b/internal/workapi/list.go @@ -194,6 +194,20 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter if err := issueops.ValidateReadyFlagScope(in); err != nil { return types.IssueFilter{}, err } + // The pinned pair is a contradiction, refused here rather than in any one + // frontend so every route agrees: the CLI refuses it at flag parse, and a + // served listing carries both members (issues.list.filters), where the + // branch below used to let PinnedFlag win silently. + if in.PinnedFlag && in.NoPinnedFlag { + return types.IssueFilter{}, fmt.Errorf("--pinned and --no-pinned are mutually exclusive%.0w", issueops.ErrValidation) + } + // The parent pair likewise, in the sentence `bd list` and BuildCountFilter + // refuse it with: a served listing carries no_parent beside parent + // (issues.list.filters), and ANDing the two answered only the edge-less + // rows under the parent's dotted id instead of refusing. + if in.ParentID != "" && in.NoParent { + return types.IssueFilter{}, fmt.Errorf("--parent and --no-parent are mutually exclusive%.0w", issueops.ErrValidation) + } filter := types.IssueFilter{ Limit: SQLLimit(in), @@ -248,6 +262,16 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter } filter.ExcludeStatus = excludeStatuses } + // The caller's own exclusions (ListRequest.ExcludeStatus) are APPENDED to + // the defaults above, never in place of them: excluding `blocked` must not + // re-admit closed rows. They apply on every arm, AllFlag and an explicit + // Status included, which is what lets a caller drop closed history + // (AllFlag with ExcludeStatus "closed") without enumerating every status. + excluded, err := ParseExcludeStatus(in.ExcludeStatus, cfg.CustomStatusNames()) + if err != nil { + return filter, err + } + filter.ExcludeStatus = append(filter.ExcludeStatus, excluded...) if in.Priority != nil { p := *in.Priority diff --git a/internal/workapi/list_pinned_test.go b/internal/workapi/list_pinned_test.go index 38e944092..02dd6ad53 100644 --- a/internal/workapi/list_pinned_test.go +++ b/internal/workapi/list_pinned_test.go @@ -1,6 +1,9 @@ package workapi import ( + "errors" + "slices" + "strings" "testing" "github.com/steveyegge/beads/internal/types" @@ -161,3 +164,72 @@ func TestStatusAllCannotBeCombined(t *testing.T) { t.Errorf("error = %q, want %q", err, want) } } + +// TestPinnedPairIsRefusedByTheBuilder pins the library half of the pinned +// pair's refusal. The CLI refuses `--pinned --no-pinned` at flag parse, but the +// builder is the one point every route passes through — a served listing +// carries both members since issues.list.filters — and it used to let +// PinnedFlag win silently. Each member alone still builds. +func TestPinnedPairIsRefusedByTheBuilder(t *testing.T) { + var cfg ListConfig + for _, status := range []string{"", "all", "open"} { + _, err := BuildListFilter(issueops.ListRequest{Status: status, PinnedFlag: true, NoPinnedFlag: true}, cfg) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("--status=%q --pinned --no-pinned: err = %v, want ErrValidation", status, err) + } + } + for _, in := range []issueops.ListRequest{{PinnedFlag: true}, {NoPinnedFlag: true}} { + if _, err := BuildListFilter(in, cfg); err != nil { + t.Errorf("BuildListFilter(%+v) = %v, want no error for one member alone", in, err) + } + } +} + +// TestParentPairIsRefusedByTheBuilder is the parent pair's twin of the pinned +// test above: the CLI refuses `--parent X --no-parent` at flag parse, a served +// listing carries both members since issues.list.filters, and the builder used +// to AND them. The sentence is the one the http handler maps to a 400 on +// no_parent, and the one BuildCountFilter refuses the same pair with. +func TestParentPairIsRefusedByTheBuilder(t *testing.T) { + var cfg ListConfig + _, err := BuildListFilter(issueops.ListRequest{ParentID: "bd-1", NoParent: true}, cfg) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("--parent bd-1 --no-parent: err = %v, want ErrValidation", err) + } + if want := "--parent and --no-parent are mutually exclusive"; err.Error() != want { + t.Errorf("--parent bd-1 --no-parent: err = %q, want %q", err, want) + } + for _, in := range []issueops.ListRequest{{ParentID: "bd-1"}, {NoParent: true}} { + if _, err := BuildListFilter(in, cfg); err != nil { + t.Errorf("BuildListFilter(%+v) = %v, want no error for one member alone", in, err) + } + } +} + +// TestListExcludeStatusAppendsAndValidates pins ListRequest.ExcludeStatus in +// the builder: it is appended to the default exclusions (never in their place), +// stands alone under AllFlag, splits commas, and an unknown name is +// ErrValidation with the count's own sentence, which the http handler maps to +// a 400 on exclude_status. +func TestListExcludeStatusAppendsAndValidates(t *testing.T) { + var cfg ListConfig + f, err := BuildListFilter(issueops.ListRequest{ExcludeStatus: []string{"blocked, in_progress"}}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + want := []types.Status{types.StatusClosed, types.StatusPinned, types.StatusBlocked, types.StatusInProgress} + if !slices.Equal(f.ExcludeStatus, want) { + t.Errorf("default listing ExcludeStatus = %v, want %v", f.ExcludeStatus, want) + } + f, err = BuildListFilter(issueops.ListRequest{AllFlag: true, ExcludeStatus: []string{"closed"}}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if want := []types.Status{types.StatusClosed}; !slices.Equal(f.ExcludeStatus, want) { + t.Errorf("AllFlag ExcludeStatus = %v, want %v", f.ExcludeStatus, want) + } + _, err = BuildListFilter(issueops.ListRequest{ExcludeStatus: []string{"clsoed"}}, cfg) + if !errors.Is(err, issueops.ErrValidation) || !strings.HasPrefix(err.Error(), "invalid exclude-status ") { + t.Errorf("unknown exclude status: err = %v, want ErrValidation starting \"invalid exclude-status \"", err) + } +} diff --git a/issueops/counter.go b/issueops/counter.go index f996cb5f4..c991f6dda 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -122,11 +122,9 @@ type CountRequest struct { // It composes with IncludeInfra's own "gate" exclusion rather than // replacing it: both contribute to the same exclusion set. ExcludeTypes []string - // ExcludeStatus names statuses to exclude. Unlike Status, this has no - // List counterpart — ListRequest computes its default status exclusions - // internally from workspace configuration and exposes no caller-facing - // knob for them. Here it is a new, Count-only capability (behavior token - // `issues.count.scope`): entries may be comma-separated, and — UNLIKE + // ExcludeStatus names statuses to exclude (behavior token + // `issues.count.scope`; ListRequest.ExcludeStatus is its listing twin, + // validated by the same rule): entries may be comma-separated, and — UNLIKE // Status and ExcludeTypes above — each name IS validated against the // workspace vocabulary (built-in statuses plus the workspace's own custom // ones). An unrecognized name is ErrValidation rather than excluding diff --git a/issueops/reader.go b/issueops/reader.go index 852a9a9c2..23cee7bd3 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -144,6 +144,16 @@ type ListRequest struct { // so `pinned` and `hooked` are the two spellings that also drop that // predicate — every other status narrows to unflagged rows. Status string + // ExcludeStatus names statuses to leave out, on top of whatever Status or + // the default exclusions select (it never re-admits a row they hide). + // Entries may be comma-separated, and each name is validated against the + // workspace vocabulary (built-in plus custom statuses) exactly as + // CountRequest.ExcludeStatus is: an unknown name is ErrValidation rather + // than excluding nothing, because a silently ignored exclusion answers a + // wider set. With AllFlag there are no default exclusions, so the + // listing keeps every status not named here, without the caller + // having to enumerate every status it wants. + ExcludeStatus []string // IssueType is validated against the workspace vocabulary — unlike // ReadyRequest.IssueType, which matches nothing rather than failing. IssueType string @@ -284,6 +294,11 @@ type ListRequest struct { // On the ReadyFlag arm neither applies: that set decides pinned for itself, // which is why PinnedFlag is refused there and NoPinnedFlag merely accepted // (see AllFlag). + // + // The two together are a contradiction and are refused with ErrValidation + // on every route, the way `bd list --pinned --no-pinned` refuses: no + // answer is both only-flagged and only-unflagged, and letting one win + // would answer a question the caller did not ask. PinnedFlag bool NoPinnedFlag bool IncludeTemplates bool @@ -391,6 +406,11 @@ type ListRequest struct { // NoParent restricts to rows with no parent-child dependency edge. It has // no dotted-id arm, so an edge-less "X.1" matches BOTH ParentID X and // NoParent. The ReadyFlag arm refuses it; see that field. + // + // Setting both is refused with ErrValidation on every route, as + // CountRequest refuses it and in the sentence `bd list --parent X + // --no-parent` refuses with, rather than answered with only the edge-less + // rows under X's dotted id, which both clauses match. ParentID string NoParent bool MolType *MolType @@ -437,7 +457,7 @@ type ListRequest struct { // fields rather than answering a wider question than was asked. That is // IDFilter, TitleSearch, SpecPrefix, the four *Contains fields, // ExternalRef, every Created/Updated/Closed/Defer/Due bound, DeferredFlag, - // OverdueFlag, EmptyDesc, NoLabels, NoParent, PinnedFlag, PriorityMin, + // OverdueFlag, EmptyDesc, NoLabels, NoParent, PinnedFlag, ExcludeStatus, PriorityMin, // PriorityMax, and the keyset position. There is no fallback: no // combination of the two silently widens the answer. // diff --git a/issueops/reader_ready_scope.go b/issueops/reader_ready_scope.go index b69bc686b..a62533298 100644 --- a/issueops/reader_ready_scope.go +++ b/issueops/reader_ready_scope.go @@ -91,6 +91,8 @@ var readyScopeFields = []readyScopeField{ {"NoLabels", "--no-labels", func(r ListRequest) bool { return r.NoLabels }}, {"NoParent", "--no-parent", func(r ListRequest) bool { return r.NoParent }}, {"PinnedFlag", "--pinned", func(r ListRequest) bool { return r.PinnedFlag }}, + // No `bd list` flag sets ExcludeStatus yet, so its wire name stands in. + {"ExcludeStatus", "exclude_status", func(r ListRequest) bool { return len(r.ExcludeStatus) > 0 }}, {"PriorityMin", "--priority-min", func(r ListRequest) bool { return r.PriorityMin != nil }}, {"PriorityMax", "--priority-max", func(r ListRequest) bool { return r.PriorityMax != nil }}, From 4450fe99bb71524f8e6f1d00d8567b0c4303b0ef Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 17 Oct 2026 12:29:13 +0000 Subject: [PATCH 09/54] feat(http): bd blocked over bd serve (blocked.list), plus bd blocked --flags [--raw] (#7649) * feat(http): bd blocked over bd serve (blocked.list) through issueops.BlockedLister `bd blocked` refused against an HTTP workspace: it called the legacy GetBlockedIssues, which the HTTP backend stubs as unsupported. It now runs one role on every route, and bd serve publishes that role. - Role: issueops.BlockedLister.ListBlocked(BlockedRequest{ParentID, Labels, LabelsAny, ExcludeLabels}) -> BlockedResult{Items}. One body, internal/storage/issueops.ExecuteListBlockedInTx over the unchanged GetBlockedIssuesInTx; labels normalized in the role (BlockedFilter); Items never nil and in a total order (priority, created_at desc, id). - Legs: DoltStore (read tx), EmbeddedDoltStore (read conn), uow (read unit of work through the use case's GetBlockedIssues, so the external-dependency uow decorator applies). Census: storage.DoltStorage gains BlockedLister(); HookFiringStore passes it through (a read), telemetry wraps it, uow.notifyingProvider, externaldeps.uowProvider and httpapi.timedProvider build it. - externaldeps.Store.BlockedLister is built over its own GetBlockedIssues, so local `bd blocked` output is unchanged. Over a remote store it takes the server's listing when the server advertises policy.external_dependencies, else decorates the server's listing here through EdgeReader and BatchGetter (decorateBlocked, factored out of GetBlockedIssues). - Wire: GET /v0/beads/blocked (listBlockedIssues; parent, label, label_any, exclude_label) answering BlockedPage {items, has_more} with the new x-go-type-pinned BlockedIssue schema, behind the new blocked.list token. apigen and the wire-shape golden regenerated; the golden diff is additive only, so wire_revision stays 2. Config.BlockedLister is a required role, wired in bd serve. - Client: httpclient.Store.BlockedLister dials through Store.dispatch, so an older server is refused BEFORE the dial with *wire.CapabilityError naming blocked.list. The legacy Store.GetBlockedIssues rides the role (page and row cap applied to its answer) and leaves the unsupported shell. Divergence ledger row F-blocked retired. - CLI: `bd blocked` takes BlockedLister() from the store on the direct routes and from the provider's BlockedListerSource on the proxied route; runBlockedProxiedServer is deleted. --json output is the same rows in the same order, [] when empty. Tests: backend/conformance RunBlockedLister (10 cases: direct blocker, child attributed to its blocked parent, closed blocker, wisp plane, pinned status, parent filter, label filters with normalization, order, empty slice, request not mutated) on dolt, embedded, uow and the served HTTP client; httpapi handler tests (mapping, unknown/repeated params, projection, capability); httpclient stub tests (pre-dial refusal naming the token, encoding, projection, legacy door paging); skew and vocabulary counts; TestE2E_BlockedOverHTTP (served `bd blocked --json` equals the local command's after mapping ids and timestamps, member order included, external blocker named, older-server refusal with no dial); corpus gc-blocked re-marked and gc-blocked-parent added, both pending until the reference server advertises blocked.list. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(blocked): pin server-side external decoration, refuse has_more, serve the legacy door over http Review fixes for the blocked.list slice. - Server-side external decoration is now pinned on the compositions bd serve answers from. TestWrapUOWProviderBlockedListerNamesExternalBlockers lists through the wrapped unit-of-work provider's own BlockedLister accessor (the bd serve provider path) and requires an external-only row and a mixed row to carry their external: refs; building that lister over the inner provider fails it. TestServedBlockedListerCarriesExternalRefs- WhenTheServerEnforces serves the store-shaped composition (externaldeps.Store.BlockedLister, policy advertised) and requires the operation's own answer, through the undecorated client role, to carry the refs, and the client's decorated chain to return it unchanged; serving the inner store's lister fails it. - The client refuses an answer reporting has_more with the new typed httpclient.TruncatedListingError instead of returning a short listing: the operation has no cursor, and a caller reads a missing row as unblocked. The legacy door inherits it. - externaldeps.Store.GetBlockedIssues over an http store reads the extra sources through the served roles (EdgeReader, BatchGetter) instead of the raw GetIssuesByIDs / GetDependencyRecordsForIssues the http backend refuses, and the decoration skips the source read when there is nothing to load. The legacy door on the CLI chain (bd mol stale, the wisp commands) now answers over http in both policy modes; TestServedLegacyGetBlockedIssuesThroughTheDecorator pins it, paging included. bd ready --explain still refuses on its other raw reads; the CHANGELOG says so. - Docs: the spec and the retired F-blocked ledger row state that the listing is unpaged, bounded by the client's 32 MiB response limit (refused, never read in part), and that has_more is refused. - Contract: new cases for waits-for and conditional-blocks blockers, no type exclusions (gate, epic), and a stale flag with no nameable blocker (on the legs that can write the column: dolt, embedded; skipped elsewhere with the reason). Order seeds explicit created_at values with five rows tied on priority and created_at, two of them wisps, so only the id tiebreak orders them; dropping the tiebreak now fails every run. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(blocked): bd blocked --flags, the blocked flag per id from stdin `bd blocked --flags [--json]` reads ids from stdin, one per line (blank lines and repeats ignored), and prints the blocked flag bd holds for each: with --json, [{"id","is_blocked"}] in the order the ids were first given, omitting an id that names no row on either plane, [] for no input. It is the readiness column a consumer otherwise had to read with `bd sql`, so no such consumer needs a local-only command. It is the FLAG, not the listing: `bd blocked` narrows the stored flag to rows it can name a blocker for, and --flags does not, so a row whose flag is set with no nameable blocker reads true here and is absent from the listing. Where the external-dependency policy applies, an unsatisfied external: blocker sets the flag, as it keeps the row out of `bd ready`. One library entry on every route: issueops.BlockedFlags over the route's BlockingAnnotator (the store's accessor on dolt, embedded and HTTP, the provider's on the proxied route). Over HTTP that is listBlockingAnnotations with is_blocked behind dependencies.blocking.isBlocked; a server without that token is refused before the dial, naming it. No wire change. The listing's filter flags are refused with --flags rather than ignored. Tests: TestE2E_BlockedFlagsOverHTTP (served JSON equals the local command's after mapping ids to titles, unknown id omitted, repeat collapsed, [] on empty stdin, filters refused; a stale-flagged row with no blocker reads true and is absent from `bd blocked --json`, which fails if --flags is derived from the listing; an older server's handshake refused naming the token with no dial); TestProxiedServerBlocked/flags_reads_ids_from_stdin on the proxied route; corpus gc-blocked-flags, pending until the reference server advertises the token. CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(blocked): bd blocked --flags --raw, the stored is_blocked column `bd blocked --flags --raw [--json]` answers the stored is_blocked column for each id read from stdin, exactly what reading is_blocked from the issues and wisps tables returns, without the external-dependency policy: a row blocked only by an unsatisfied external: ref reads false (the policy-aware --flags reads true), and a stale flag reads true either way. Same output shape as --flags. This lets a consumer move off a `bd sql` read of the column with no change in what it reads. The default --flags is unchanged. The role is taken from beneath every decorator: storage.UnwrapStore on the direct routes, uow.UnwrapProvider on the proxied route. Over HTTP the innermost store is the http client, whose annotation read is the server's answer: against a server that does not advertise policy.external_dependencies that answer is the column, and --raw is served. A server that advertises it decorates the flag it answers, the v0 wire has no member asking it for the column, and the decorated flag cannot be turned back into the column client-side, so --raw against such a server is refused before the annotation read, naming the token, rather than answered with the decorated flag. --raw without --flags is refused. Tests: TestE2E_BlockedFlagsOverHTTP/raw_is_the_stored_column (served --raw equals local --raw; the externally-blocked-only row reads true with --flags and false with --raw on both; locally blocked rows agree; a stale-flagged row reads true in both modes; --raw without --flags exits 1; a server advertising the policy refuses --raw with no annotation request); TestProxiedServerBlocked/flags_raw_is_the_stored_column; corpus gc-blocked-flags-raw (pending until the reference server advertises dependencies.blocking.isBlocked). Routing --raw through the decorated annotator fails the e2e. CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(changelog): bd blocked --flags --raw is for the local and proxied routes Co-Authored-By: Claude Opus 5.5 (1M context) * test(blocked): give the e2e gate-kid its own priority; document --flags answers closed ids TestE2E_BlockedOverHTTP seeded gate-parent and gate-kid at one priority, so when both landed in the same second their order fell to the generated ids, which differ between the local and the served workspace: the comparison failed intermittently. gate-kid now has its own priority. Ten runs on the embedded lane pass. bd blocked --flags answers a closed id too, with its stored flag; a SQL read of the column filtered on status <> 'closed' leaves it out. Said in the command's doc and the CHANGELOG. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(blocked): apply label filters to external-only blocked rows The external-dependency policy appends a row blocked only by an unsatisfied external: ref after the label query has run, so --label, --label-any and --exclude-label never judged it: bd blocked and GET /v0/beads/blocked listed it whatever its labels, on every route. Both decorators now filter those rows by the request's labels. The uow decorator reads them through GetMany, chunked at MaxGetManyIDs, so their labels are hydrated; GetIssuesByIDs left them empty, which also printed them unlabelled on the proxied route. Review fixes (adopt-pr apply-fixes, iteration 1): - major #1: label filters on external-only rows in the store and uow decorators, with absence-asserting arms in the unit, served httpclient (server- and client-enforced policy), e2e and proxied tiers; the spec and the role doc say such rows are narrowed like every other row. - minor #2: the policy.external_dependencies token prose names the blocked listing (spec, routes.go, regenerated types.gen.go). - minor #6: SERVE_RUNBOOK and the allowUnlimited comment record the blocked listing's unbounded whole-set answer and why. - minor #10: BlockedLister telemetry-outermost and typed-unsupported pins beside the ReadyCounter pair, plus the externaldeps wrap pin. - nit #9: F-blocked names --flags --raw's refusal against a server that applies the policy (ledger.go and the regenerated ledger doc). - nit #11: the remote BlockedLister no longer builds a local lister it then discards. Review fixes (adopt-pr apply-fixes, iteration 2; the review approved at 940 with one minor and two nits, none gating): - minor (the --flags refusal list was unpinned): the new TestBlockedFlagsRefusesEveryListingFilter pins blockedFlagFilterFlags to the flags blockedCmd registers, in both directions. Every registered flag except --flags and --raw must be refused, and every entry must name a registered flag. The test walks Flags() rather than LocalFlags(), so it does not merge the root's persistent flags into blockedCmd, and it skips inherited root flags and cobra's --help. Mutants: - four fail it: a new filter flag left off the list, a dropped entry, a misspelled entry, and --raw added to the list; - a simulated earlier in-process run (merged root flags plus --help) stays green, and fails once either skip is removed. - nit (the remote branch's span): the remote BlockedLister comment now says why that lister is not wrapped again. The inner lister already carries telemetry's span, so wrapping would nest a second span of the same name. The comment also says what runs outside that span. - nit (duplicated order comparator): left as is. finishBlockedIssues predates this PR, so merging the two sorts would refactor existing code with no review pass left after this one. Co-Authored-By: Claude Opus 5.5 Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 56 +++ backend/conformance/BUILD.bazel | 1 + .../conformance/blocked_lister_contract.go | 426 ++++++++++++++++++ backend/http/BUILD.bazel | 2 + backend/http/corpus/harness_test.go | 2 + backend/http/corpus/testdata/corpus.yaml | 18 +- backend/http/e2e_blocked_flags_test.go | 287 ++++++++++++ backend/http/e2e_blocked_test.go | 229 ++++++++++ backend/http/e2e_cli_test.go | 2 + cmd/bd/BUILD.bazel | 2 + cmd/bd/blocked_embedded_test.go | 2 +- cmd/bd/blocked_flags.go | 167 +++++++ cmd/bd/blocked_flags_test.go | 57 +++ cmd/bd/blocked_proxied_integration_test.go | 102 +++++ cmd/bd/ready.go | 78 ++-- cmd/bd/ready_proxied_server.go | 49 +- cmd/bd/serve.go | 4 + cmd/bd/serve_source_test.go | 1 + cmd/bd/serve_store_identity_test.go | 6 + cmd/bd/stale.go | 2 +- engdocs/SERVE_RUNBOOK.md | 8 + engdocs/design/http-divergence-ledger.md | 4 +- internal/httpapi/BUILD.bazel | 2 + internal/httpapi/apigen/types.gen.go | 31 +- internal/httpapi/blocked.go | 72 +++ internal/httpapi/blocked_test.go | 150 ++++++ internal/httpapi/claim.go | 7 + internal/httpapi/pinning.go | 4 + internal/httpapi/problem.go | 7 + internal/httpapi/reads.go | 5 + internal/httpapi/roles_test.go | 29 ++ internal/httpapi/routes.go | 18 +- internal/httpapi/server.go | 24 +- internal/httpapi/spec/openapi.v0.yaml | 226 +++++++++- internal/httpapi/wire_bijection_test.go | 3 +- .../httpapi/wireshape/testdata/golden.json | 418 +++++++++++++++++ internal/httpclient/BUILD.bazel | 4 + internal/httpclient/accessors.go | 8 + internal/httpclient/blockedlister.go | 130 ++++++ internal/httpclient/blockedlister_test.go | 230 ++++++++++ internal/httpclient/completeness_test.go | 1 - internal/httpclient/encode/ledger.go | 12 +- internal/httpclient/encode/oracle_test.go | 2 + .../served_blocked_external_test.go | 283 ++++++++++++ .../httpclient/served_blocked_lister_test.go | 24 + internal/httpclient/served_harness_test.go | 2 + internal/httpclient/served_test.go | 3 + internal/httpclient/unsupported_gen.go | 7 +- internal/httpclient/wire/handshake.go | 5 + internal/httpclient/wire/paths.go | 2 + internal/httpclient/wire/skew_test.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 1 + internal/storage/BUILD.bazel | 1 + internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/blocked_lister.go | 38 ++ .../dolt/blocked_lister_contract_test.go | 40 ++ internal/storage/embeddeddolt/BUILD.bazel | 2 + .../storage/embeddeddolt/blocked_lister.go | 40 ++ .../blocked_lister_contract_test.go | 46 ++ internal/storage/externaldeps/BUILD.bazel | 2 + .../storage/externaldeps/blocked_lister.go | 114 +++++ .../externaldeps/blocked_lister_test.go | 176 ++++++++ internal/storage/externaldeps/decorator.go | 67 ++- .../storage/externaldeps/uow_decorator.go | 22 +- .../externaldeps/uow_decorator_test.go | 33 +- internal/storage/hook_blocked_lister.go | 14 + internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/blocked_role.go | 114 +++++ .../storage/role_accessor_decorator_test.go | 18 +- internal/storage/storage.go | 6 + internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/blocked_lister.go | 50 ++ .../uow/blocked_lister_contract_test.go | 31 ++ internal/storage/uow/notifying.go | 7 + internal/telemetry/BUILD.bazel | 1 + internal/telemetry/blocked_lister.go | 38 ++ .../telemetry/role_accessor_decorator_test.go | 42 +- issue_roles_external_test.go | 30 ++ issueops/BUILD.bazel | 1 + issueops/blockedlister.go | 64 +++ 81 files changed, 4084 insertions(+), 146 deletions(-) create mode 100644 backend/conformance/blocked_lister_contract.go create mode 100644 backend/http/e2e_blocked_flags_test.go create mode 100644 backend/http/e2e_blocked_test.go create mode 100644 cmd/bd/blocked_flags.go create mode 100644 cmd/bd/blocked_flags_test.go create mode 100644 internal/httpapi/blocked.go create mode 100644 internal/httpapi/blocked_test.go create mode 100644 internal/httpclient/blockedlister.go create mode 100644 internal/httpclient/blockedlister_test.go create mode 100644 internal/httpclient/served_blocked_external_test.go create mode 100644 internal/httpclient/served_blocked_lister_test.go create mode 100644 internal/storage/dolt/blocked_lister.go create mode 100644 internal/storage/dolt/blocked_lister_contract_test.go create mode 100644 internal/storage/embeddeddolt/blocked_lister.go create mode 100644 internal/storage/embeddeddolt/blocked_lister_contract_test.go create mode 100644 internal/storage/externaldeps/blocked_lister.go create mode 100644 internal/storage/externaldeps/blocked_lister_test.go create mode 100644 internal/storage/hook_blocked_lister.go create mode 100644 internal/storage/issueops/blocked_role.go create mode 100644 internal/storage/uow/blocked_lister.go create mode 100644 internal/storage/uow/blocked_lister_contract_test.go create mode 100644 internal/telemetry/blocked_lister.go create mode 100644 issueops/blockedlister.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index c48bdeb79..8cc9f773e 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -176,6 +176,7 @@ 40 3 TestTransactionCleanupBeforeSQLCommitRemainsDefinite 40 3 TestUnknownProvenanceLeaseStaysReclaimable +40 4 TestBlockedListerContract 40 4 TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded 40 4 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering 40 4 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay diff --git a/CHANGELOG.md b/CHANGELOG.md index 99789418d..c9bb95944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `bd blocked --flags` prints the blocked flag for each id read from stdin + (one per line; blank lines and repeats ignored), with `--json` as + `[{"id","is_blocked"}]` in the order the ids were given, omitting an id that + names no row on either plane. A closed id is answered too, with its stored + flag (a `status <> 'closed'` SQL read of the column leaves it out). It is the flag `bd ready` filters on, not the + listing: a row whose stored flag is set with no nameable blocker reads true + here and is absent from `bd blocked`. It runs `issueops.BlockedFlags` on + every route; over HTTP that is `listBlockingAnnotations` with `is_blocked`, + behind `dependencies.blocking.isBlocked`, refused before the dial against a + server that does not advertise it. The listing's filter flags are refused + with `--flags`. +- `bd blocked --flags --raw` answers the stored `is_blocked` column itself, + without the external-dependency policy: a row blocked only by an + unsatisfied `external:` ref reads `false` (the policy-aware `--flags` + reads `true`); a stale flag reads `true` either way. `--raw` is for the + local and proxied routes, where it reads the column directly and the two + answer the same. Over HTTP it is answered only by a server that does not + advertise `policy.external_dependencies` (whose flag is the column); a + server advertising it answers only the policy-aware flag, so `--raw` + against it is refused before the dial rather than answered with the + decorated flag. `--raw` without `--flags` is refused. +- `bd blocked` works over a connected HTTP workspace. The blocked listing is + a role, `issueops.BlockedLister` (`ListBlocked(BlockedRequest{ParentID, + Labels, LabelsAny, ExcludeLabels}) -> BlockedResult{Items}`), on every + backend leg over the same body `GetBlockedIssues` runs, and `bd serve` + publishes it as `GET /v0/beads/blocked` (`parent`, and the repeatable + `label`, `label_any` and `exclude_label`), answering `{items, has_more}` + with `BlockedIssue` rows (the element `bd blocked --json` prints), behind + the new `blocked.list` capability token (additive; no wire_revision bump). + The listing names unsatisfied `external:` blockers wherever the + external-dependency policy applies: on the server when it advertises + `policy.external_dependencies`, else on the client, as every other policy + read does. The HTTP backend's `BlockedLister()` and legacy + `GetBlockedIssues` dial it; against a server that does not advertise the + token they refuse before dialing with a typed capability error naming + `blocked.list`, and never compose the listing from other reads. The + listing is unpaged: a server answer that reports `has_more` is refused + with the new `httpclient.TruncatedListingError` rather than returned + short, and an answer over the client's 32 MiB response limit is refused + as any oversized response is. The external-dependency decorator's legacy + `GetBlockedIssues` over an HTTP store now reads the extra sources through + the served roles (it failed on a raw multi-id read before), so callers of + that door no longer fail on it; commands that also make other raw reads + the HTTP backend does not serve (`bd ready --explain`) still refuse on + those. `bd + blocked` runs the role on every route (the proxied-server route through + the provider's accessor) with unchanged output: `--json` prints the same + rows in the same order (priority, then newest first, then id, which now + also breaks a tie the order used to leave to chance), and `[]`, never + `null`, when nothing is blocked. - `bd list`'s remaining filters work over a connected HTTP workspace. `GET /v0/beads/issues` accepts `updated_after`/`updated_before`, `closed_after`/`closed_before`, `defer_after`/`defer_before`, @@ -678,6 +728,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd blocked --label`, `--label-any` and `--exclude-label` apply to a row + blocked only by an unsatisfied `external:` dependency.** The + external-dependency policy lists such a row itself, after the label query + has run, and it was listed whatever its labels; it is now narrowed by the + label filters like every other row, on every route. On the proxied-server + route such a row also printed with no labels; it now prints them. - **`bd gate add-waiter` survives contention the store gave up replaying.** It retried only a version miss, so a convoy adder failed when its guarded write ended in a serialization failure or deadlock the backend stopped diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index d491cd407..abf0ea1ed 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "batch_closer_contract.go", "batch_creator_contract.go", "batch_getter_contract.go", + "blocked_lister_contract.go", "blocked_state.go", "blocking_annotator_contract.go", "bootstrapper_contract.go", diff --git a/backend/conformance/blocked_lister_contract.go b/backend/conformance/blocked_lister_contract.go new file mode 100644 index 000000000..9313e3e76 --- /dev/null +++ b/backend/conformance/blocked_lister_contract.go @@ -0,0 +1,426 @@ +package conformance + +import ( + "context" + "fmt" + "reflect" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the semantic contract every implementation of +// publicops.BlockedLister must satisfy: what issueops/blockedlister.go +// promises, asserted on every leg (dolt, embedded dolt, unit of work, and the +// served HTTP client) so `bd blocked` answers one listing whichever route it +// takes. +// +// EVERY CASE IS SCOPED BY A LABEL: the blocked set is a property of the whole +// database, not of an id prefix, so a row seeded by one case is inside the +// next one's answer unless each asks through Labels. + +// BlockedListerFixture supplies adapter-specific storage access for the +// blocked-listing assertions. +type BlockedListerFixture struct { + // IssuePrefix namespaces the ids each case seeds, so several cases can + // share one database. + IssuePrefix string + // BlockedLister is the surface under test. + BlockedLister publicops.BlockedLister + // CreateIssue seeds a durable issue in the issues plane. + CreateIssue func(context.Context, *types.Issue, string) error + // CreateWisp seeds an ephemeral issue in the wisps plane. + CreateWisp func(context.Context, *types.Issue, string) error + // AddDependency seeds ONE edge, recomputing the stored blocked flag the + // way every dependency write does. + AddDependency func(context.Context, *types.Dependency, string) error + // MarkBlocked sets the stored is_blocked flag on one durable issue + // directly, without the edge that would justify it: the stale flag `bd + // recompute-blocked` exists to repair. It must fail unless exactly one row + // changed. A nil hook means this leg cannot write the column, and the case + // that needs it skips with that reason. + MarkBlocked func(context.Context, string) error +} + +// RunBlockedLister runs every blocked-listing case as a subtest of t. +func RunBlockedLister(t *testing.T, ctx context.Context, fixture BlockedListerFixture) { + t.Helper() + t.Run("NamesADirectBlocker", func(t *testing.T) { runBlockedNamesADirectBlocker(t, ctx, fixture) }) + t.Run("AttributesAChildToItsBlockedParent", func(t *testing.T) { runBlockedAttributesAChildToItsParent(t, ctx, fixture) }) + t.Run("ClosedBlockerDoesNotBlock", func(t *testing.T) { runBlockedClosedBlockerDoesNotBlock(t, ctx, fixture) }) + t.Run("IncludesTheWispPlane", func(t *testing.T) { runBlockedIncludesTheWispPlane(t, ctx, fixture) }) + t.Run("ExcludesThePinnedStatus", func(t *testing.T) { runBlockedExcludesThePinnedStatus(t, ctx, fixture) }) + t.Run("ParentFilter", func(t *testing.T) { runBlockedParentFilter(t, ctx, fixture) }) + t.Run("LabelFilters", func(t *testing.T) { runBlockedLabelFilters(t, ctx, fixture) }) + t.Run("Order", func(t *testing.T) { runBlockedOrder(t, ctx, fixture) }) + t.Run("NamesWaitsForAndConditionalBlockers", func(t *testing.T) { runBlockedNamesWaitsForAndConditionalBlockers(t, ctx, fixture) }) + t.Run("OmitsAStaleFlagWithNoNameableBlocker", func(t *testing.T) { runBlockedOmitsAStaleFlag(t, ctx, fixture) }) + t.Run("HasNoTypeExclusions", func(t *testing.T) { runBlockedHasNoTypeExclusions(t, ctx, fixture) }) + t.Run("EmptyIsAnEmptySlice", func(t *testing.T) { runBlockedEmptyIsAnEmptySlice(t, ctx, fixture) }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { runBlockedDoesNotMutateTheRequest(t, ctx, fixture) }) +} + +// runBlockedNamesADirectBlocker: an issue with an open `blocks` target is +// listed with that target as its one blocker; the target itself is not. +func runBlockedNamesADirectBlocker(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-direct" + blocked, blocker := blListerID(f, "direct-src"), blListerID(f, "direct-dst") + blListerSeedIssue(t, ctx, f, blListerIssue(blocked, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label)) + blListerSeedEdge(t, ctx, f, blocked, blocker, types.DepBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{blocked: {blocker}}) +} + +// runBlockedAttributesAChildToItsParent: a child of a blocked parent, with no +// blocker of its own, is listed with its parent named as the blocker. +func runBlockedAttributesAChildToItsParent(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-child" + parent, child, blocker := blListerID(f, "child-parent"), blListerID(f, "child-child"), blListerID(f, "child-blocker") + blListerSeedIssue(t, ctx, f, blListerIssue(parent, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label+"-other")) + blListerSeedIssue(t, ctx, f, blListerIssue(child, 1, label)) + blListerSeedEdge(t, ctx, f, child, parent, types.DepParentChild) + blListerSeedEdge(t, ctx, f, parent, blocker, types.DepBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{parent: {blocker}, child: {parent}}) +} + +// runBlockedClosedBlockerDoesNotBlock: a `blocks` edge onto a closed target +// names no blocker, so the source is not listed. +func runBlockedClosedBlockerDoesNotBlock(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-closed" + src, dst := blListerID(f, "closed-src"), blListerID(f, "closed-dst") + blListerSeedIssue(t, ctx, f, blListerIssue(src, 1, label)) + closed := blListerIssue(dst, 1, label) + closed.Status = types.StatusClosed + blListerSeedIssue(t, ctx, f, closed) + blListerSeedEdge(t, ctx, f, src, dst, types.DepBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{}) +} + +// runBlockedIncludesTheWispPlane: a blocked wisp is listed beside the issues. +func runBlockedIncludesTheWispPlane(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-wisp" + wisp, blocker := blListerID(f, "wisp-src"), blListerID(f, "wisp-dst") + w := blListerIssue(wisp, 1, label) + w.Ephemeral = true + if err := f.CreateWisp(ctx, w, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", wisp, err) + } + b := blListerIssue(blocker, 1, label) + b.Ephemeral = true + if err := f.CreateWisp(ctx, b, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", blocker, err) + } + blListerSeedEdge(t, ctx, f, wisp, blocker, types.DepBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{wisp: {blocker}}) +} + +// runBlockedExcludesThePinnedStatus: a row whose status is pinned is never +// listed, whatever blocks it. +func runBlockedExcludesThePinnedStatus(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-pinned" + pinned, open, blocker := blListerID(f, "pinned-src"), blListerID(f, "pinned-open"), blListerID(f, "pinned-dst") + p := blListerIssue(pinned, 1, label) + p.Status = types.StatusPinned + blListerSeedIssue(t, ctx, f, p) + blListerSeedIssue(t, ctx, f, blListerIssue(open, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label+"-other")) + blListerSeedEdge(t, ctx, f, pinned, blocker, types.DepBlocks) + blListerSeedEdge(t, ctx, f, open, blocker, types.DepBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{open: {blocker}}) +} + +// runBlockedParentFilter: ParentID narrows to the parent's direct children. +func runBlockedParentFilter(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-parent" + parent, kid, stranger, blocker := blListerID(f, "pf-parent"), blListerID(f, "pf-kid"), blListerID(f, "pf-stranger"), blListerID(f, "pf-blocker") + blListerSeedIssue(t, ctx, f, blListerIssue(parent, 1, label+"-other")) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label+"-other")) + blListerSeedIssue(t, ctx, f, blListerIssue(kid, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(stranger, 1, label)) + blListerSeedEdge(t, ctx, f, kid, parent, types.DepParentChild) + blListerSeedEdge(t, ctx, f, kid, blocker, types.DepBlocks) + blListerSeedEdge(t, ctx, f, stranger, blocker, types.DepBlocks) + + all := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, all, map[string][]string{kid: {blocker}, stranger: {blocker}}) + narrowed := blListerList(t, ctx, f, publicops.BlockedRequest{ParentID: parent, Labels: []string{label}}) + blListerAssertRows(t, narrowed, map[string][]string{kid: {blocker}}) +} + +// runBlockedLabelFilters: Labels is AND, LabelsAny is OR, ExcludeLabels +// removes, and every value is trimmed with blanks dropped. +func runBlockedLabelFilters(t *testing.T, ctx context.Context, f BlockedListerFixture) { + base := f.IssuePrefix + "-bl-lab" + red, blue := base+"-red", base+"-blue" + both, onlyRed, onlyBlue, blocker := blListerID(f, "lab-both"), blListerID(f, "lab-red"), blListerID(f, "lab-blue"), blListerID(f, "lab-blocker") + blListerSeedIssue(t, ctx, f, blListerIssue(both, 1, base, red, blue)) + blListerSeedIssue(t, ctx, f, blListerIssue(onlyRed, 1, base, red)) + blListerSeedIssue(t, ctx, f, blListerIssue(onlyBlue, 1, base, blue)) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, base+"-other")) + for _, id := range []string{both, onlyRed, onlyBlue} { + blListerSeedEdge(t, ctx, f, id, blocker, types.DepBlocks) + } + want := func(ids ...string) map[string][]string { + m := map[string][]string{} + for _, id := range ids { + m[id] = []string{blocker} + } + return m + } + + blListerAssertRows(t, blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{" " + red + " ", "", blue}}), want(both)) + blListerAssertRows(t, blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{base}, LabelsAny: []string{red, blue}}), want(both, onlyRed, onlyBlue)) + blListerAssertRows(t, blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{base}, ExcludeLabels: []string{" " + blue}}), want(onlyRed)) +} + +// runBlockedOrder: priority ascending, then created_at descending, then id. +// +// The created_at values are seeded explicitly, so every key decides at least +// one pair: two rows share a priority and differ in created_at (newest first), +// and five share both priority and created_at (the id breaks the tie). The +// expected order is written out rather than recomputed from the answer. +func runBlockedOrder(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-order" + blocker := blListerID(f, "order-blocker") + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label+"-other")) + older := time.Date(2026, 1, 1, 10, 0, 0, 0, time.UTC) + newer := older.Add(time.Hour) + // Two of the five tied rows are wisps. The store body assembles rows + // issues-plane first, each plane in id order, so the ids alone would put + // the tied run in the wrong order (b, d, e, then a, c); only the id + // tiebreak puts it right. + rows := []struct { + name string + priority int + created time.Time + wisp bool + }{ + {"order-p3", 3, newer, false}, + {"order-p0-older", 0, older, false}, + {"order-p0-newer-e", 0, newer, false}, + {"order-p0-newer-c", 0, newer, true}, + {"order-p0-newer-a", 0, newer, true}, + {"order-p0-newer-d", 0, newer, false}, + {"order-p0-newer-b", 0, newer, false}, + {"order-p1", 1, older, false}, + } + for _, row := range rows { + issue := blListerIssue(blListerID(f, row.name), row.priority, label) + issue.CreatedAt = row.created + issue.UpdatedAt = row.created + if row.wisp { + issue.Ephemeral = true + if err := f.CreateWisp(ctx, issue, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", issue.ID, err) + } + } else { + blListerSeedIssue(t, ctx, f, issue) + } + blListerSeedEdge(t, ctx, f, issue.ID, blocker, types.DepBlocks) + } + want := []string{ + blListerID(f, "order-p0-newer-a"), + blListerID(f, "order-p0-newer-b"), + blListerID(f, "order-p0-newer-c"), + blListerID(f, "order-p0-newer-d"), + blListerID(f, "order-p0-newer-e"), + blListerID(f, "order-p0-older"), + blListerID(f, "order-p1"), + blListerID(f, "order-p3"), + } + got := blListerIDs(blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}})) + if !slices.Equal(got, want) { + t.Errorf("listing order = %v, want %v (priority ascending, created_at descending, id)", got, want) + } +} + +// runBlockedNamesWaitsForAndConditionalBlockers: a waits-for onto a spawner +// whose child is still open, and a conditional-blocks onto an open target, +// each name their target, as a blocks edge does. +func runBlockedNamesWaitsForAndConditionalBlockers(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-wf" + spawner, spawned := blListerID(f, "wf-spawner"), blListerID(f, "wf-spawned") + waiter := blListerID(f, "wf-waiter") + target, conditional := blListerID(f, "cb-target"), blListerID(f, "cb-src") + blListerSeedIssue(t, ctx, f, blListerIssue(spawner, 1, label+"-other")) + blListerSeedIssue(t, ctx, f, blListerIssue(spawned, 1, label+"-other")) + blListerSeedIssue(t, ctx, f, blListerIssue(target, 1, label+"-other")) + blListerSeedIssue(t, ctx, f, blListerIssue(waiter, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(conditional, 1, label)) + blListerSeedEdge(t, ctx, f, spawned, spawner, types.DepParentChild) + blListerSeedEdge(t, ctx, f, waiter, spawner, types.DepWaitsFor) + blListerSeedEdge(t, ctx, f, conditional, target, types.DepConditionalBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{waiter: {spawner}, conditional: {target}}) +} + +// runBlockedOmitsAStaleFlag: a row whose stored flag says blocked but which +// has no active blocker and no parent names no blocker, so it is not listed. +// This is the narrowing gc's readiness projection documents relying on. +func runBlockedOmitsAStaleFlag(t *testing.T, ctx context.Context, f BlockedListerFixture) { + if f.MarkBlocked == nil { + t.Skip("this leg cannot write the stored is_blocked flag directly") + } + label := f.IssuePrefix + "-bl-stale" + stale, real, blocker := blListerID(f, "stale-src"), blListerID(f, "stale-real"), blListerID(f, "stale-blocker") + blListerSeedIssue(t, ctx, f, blListerIssue(stale, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(real, 1, label)) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label+"-other")) + blListerSeedEdge(t, ctx, f, real, blocker, types.DepBlocks) + if err := f.MarkBlocked(ctx, stale); err != nil { + t.Fatalf("mark %s blocked: %v", stale, err) + } + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{real: {blocker}}) +} + +// runBlockedHasNoTypeExclusions: a gate is listed like any other row; the +// blocked door excludes no issue type (unlike the ready front). +func runBlockedHasNoTypeExclusions(t *testing.T, ctx context.Context, f BlockedListerFixture) { + label := f.IssuePrefix + "-bl-type" + gate, epic, blocker := blListerID(f, "type-gate"), blListerID(f, "type-epic"), blListerID(f, "type-blocker") + g := blListerIssue(gate, 1, label) + g.IssueType = types.TypeGate + g.AwaitType = "human" + blListerSeedIssue(t, ctx, f, g) + e := blListerIssue(epic, 1, label) + e.IssueType = types.TypeEpic + blListerSeedIssue(t, ctx, f, e) + blListerSeedIssue(t, ctx, f, blListerIssue(blocker, 1, label+"-other")) + blListerSeedEdge(t, ctx, f, gate, blocker, types.DepBlocks) + blListerSeedEdge(t, ctx, f, epic, blocker, types.DepBlocks) + + items := blListerList(t, ctx, f, publicops.BlockedRequest{Labels: []string{label}}) + blListerAssertRows(t, items, map[string][]string{gate: {blocker}, epic: {blocker}}) +} + +// runBlockedEmptyIsAnEmptySlice: nothing blocked is an empty, non-nil slice +// and a nil error. +func runBlockedEmptyIsAnEmptySlice(t *testing.T, ctx context.Context, f BlockedListerFixture) { + result, err := f.BlockedLister.ListBlocked(ctx, publicops.BlockedRequest{Labels: []string{f.IssuePrefix + "-bl-nothing"}}) + if err != nil { + t.Fatalf("ListBlocked: %v", err) + } + if result.Items == nil || len(result.Items) != 0 { + t.Fatalf("Items = %#v, want an empty non-nil slice", result.Items) + } +} + +// runBlockedDoesNotMutateTheRequest: the caller's label slices come back as +// they were sent, normalization notwithstanding. +func runBlockedDoesNotMutateTheRequest(t *testing.T, ctx context.Context, f BlockedListerFixture) { + req := publicops.BlockedRequest{ + ParentID: " ", + Labels: []string{" a ", "", "a"}, + LabelsAny: []string{" b"}, + ExcludeLabels: []string{"c "}, + } + before := publicops.BlockedRequest{ + ParentID: req.ParentID, + Labels: slices.Clone(req.Labels), + LabelsAny: slices.Clone(req.LabelsAny), + ExcludeLabels: slices.Clone(req.ExcludeLabels), + } + if _, err := f.BlockedLister.ListBlocked(ctx, req); err != nil { + t.Fatalf("ListBlocked: %v", err) + } + if !reflect.DeepEqual(req, before) { + t.Errorf("request after = %#v, before = %#v", req, before) + } +} + +func blListerID(f BlockedListerFixture, name string) string { + return fmt.Sprintf("%s-%s", f.IssuePrefix, name) +} + +func blListerIssue(id string, priority int, labels ...string) *types.Issue { + return &types.Issue{ + ID: id, + Title: id, + Status: types.StatusOpen, + Priority: priority, + IssueType: types.TypeTask, + Labels: labels, + } +} + +func blListerSeedIssue(t *testing.T, ctx context.Context, f BlockedListerFixture, issue *types.Issue) { + t.Helper() + if err := f.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed issue %s: %v", issue.ID, err) + } +} + +func blListerSeedEdge(t *testing.T, ctx context.Context, f BlockedListerFixture, from, to string, kind types.DependencyType) { + t.Helper() + if err := f.AddDependency(ctx, &types.Dependency{IssueID: from, DependsOnID: to, Type: kind}, "seed"); err != nil { + t.Fatalf("seed %s edge %s -> %s: %v", kind, from, to, err) + } +} + +func blListerList(t *testing.T, ctx context.Context, f BlockedListerFixture, req publicops.BlockedRequest) []*types.BlockedIssue { + t.Helper() + result, err := f.BlockedLister.ListBlocked(ctx, req) + if err != nil { + t.Fatalf("ListBlocked(%+v): %v", req, err) + } + if result.Items == nil { + t.Fatalf("ListBlocked(%+v) Items is nil; the role promises a non-nil slice", req) + } + return result.Items +} + +func blListerIDs(items []*types.BlockedIssue) []string { + ids := make([]string, 0, len(items)) + for _, item := range items { + ids = append(ids, item.ID) + } + return ids +} + +// assertBlockedRows requires the listing to be exactly want: the same ids, +// each naming exactly the blockers want gives it (in any order), with +// BlockedByCount equal to their number. +func blListerAssertRows(t *testing.T, items []*types.BlockedIssue, want map[string][]string) { + t.Helper() + got := map[string][]string{} + for _, item := range items { + if item == nil { + t.Fatalf("listing carries a nil row") + } + if _, dup := got[item.ID]; dup { + t.Errorf("row %s listed twice", item.ID) + } + blockers := slices.Clone(item.BlockedBy) + slices.Sort(blockers) + got[item.ID] = blockers + if item.BlockedByCount != len(item.BlockedBy) { + t.Errorf("row %s: BlockedByCount = %d, len(BlockedBy) = %d", item.ID, item.BlockedByCount, len(item.BlockedBy)) + } + } + normalized := map[string][]string{} + for id, blockers := range want { + b := slices.Clone(blockers) + slices.Sort(b) + normalized[id] = b + } + if !reflect.DeepEqual(got, normalized) { + t.Errorf("listing = %v, want %v", got, normalized) + } +} diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index da2d2454d..1e622733c 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -31,6 +31,8 @@ go_test( timeout = "long", srcs = [ "attach_test.go", + "e2e_blocked_flags_test.go", + "e2e_blocked_test.go", "e2e_cli_test.go", "e2e_export_test.go", "e2e_gates_test.go", diff --git a/backend/http/corpus/harness_test.go b/backend/http/corpus/harness_test.go index ca00eea59..d2a2c4d02 100644 --- a/backend/http/corpus/harness_test.go +++ b/backend/http/corpus/harness_test.go @@ -144,6 +144,8 @@ func corpusServeConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { must("LeaseReclaimer") cfg.Heartbeater, err = s.Heartbeater() must("Heartbeater") + cfg.BlockedLister, err = s.BlockedLister() + must("BlockedLister") cfg.Relations, err = s.IssueRelations() must("IssueRelations") cfg.Commenter, err = s.Commenter() diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 788ea038f..925d1db64 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -404,7 +404,23 @@ shapes: - id: gc-blocked source: gc bdstore_ready_projection.go fetchReadyProjectionViaBlocked args: [blocked, --json] - pending: {slice: "S18", reason: "bd blocked hits the GetBlockedIssues stub (B3 #10)"} + pending: {slice: "SRV-1", reason: "bd blocked dials listBlockedIssues behind blocked.list, which the reference server does not advertise, so the client refuses at its capability preflight (pinned passing against the current server by backend/http TestE2E_BlockedOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: gc-blocked-flags + source: gc bdstore_ready_projection.go readiness door (bd blocked --flags --json, ids on stdin) + args: [blocked, --flags, --json] + stdin: "e2e-mol\ne2e-mol.1\ne2e-mol.2\ne2e-no-such-row\n" + compare: json + pending: {slice: "SRV-1", reason: "bd blocked --flags reads listBlockingAnnotations is_blocked behind dependencies.blocking.isBlocked, which the reference server does not advertise, so the client refuses at its capability preflight (pinned passing against the current server by backend/http TestE2E_BlockedFlagsOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: gc-blocked-flags-raw + source: gc bdstore_ready_projection.go readiness door (the stored is_blocked column, bd blocked --flags --raw --json, ids on stdin) + args: [blocked, --flags, --raw, --json] + stdin: "e2e-mol\ne2e-mol.1\ne2e-mol.2\ne2e-no-such-row\n" + compare: json + pending: {slice: "SRV-1", reason: "bd blocked --flags reads listBlockingAnnotations is_blocked behind dependencies.blocking.isBlocked, which the reference server does not advertise, so the client refuses at its capability preflight (pinned passing against the current server by backend/http TestE2E_BlockedFlagsOverHTTP); remove the mark when reference_server.json is upgraded"} + - id: gc-blocked-parent + source: bd blocked --parent (the molecule-scoped blocked listing) + args: [blocked, --json, --parent, e2e-mol] + pending: {slice: "SRV-1", reason: "bd blocked dials listBlockedIssues behind blocked.list, which the reference server does not advertise, so the client refuses at its capability preflight (pinned passing against the current server by backend/http TestE2E_BlockedOverHTTP); remove the mark when reference_server.json is upgraded"} # ---------------------------------------------------------------- molecules - id: mc-mol-current diff --git a/backend/http/e2e_blocked_flags_test.go b/backend/http/e2e_blocked_flags_test.go new file mode 100644 index 000000000..1943bd676 --- /dev/null +++ b/backend/http/e2e_blocked_flags_test.go @@ -0,0 +1,287 @@ +//go:build cgo + +package bdhttp_test + +import ( + "context" + "encoding/json" + "errors" + "os/exec" + "reflect" + "sort" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// runBDStdin is runBD with stdin. +func runBDStdin(t *testing.T, f *remoteRolesFixture, stdin string, args ...string) bdResult { + t.Helper() + cmd := exec.Command(f.bin, args...) + cmd.Dir = f.dir + cmd.Env = minimalBDEnv(f.dir) + cmd.Stdin = strings.NewReader(stdin) + var o, e strings.Builder + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + code := 0 + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("run bd %v: %v", args, err) + } + return bdResult{stdout: o.String(), stderr: e.String(), code: code} +} + +type blockedFlagRow struct { + ID string `json:"id"` + IsBlocked bool `json:"is_blocked"` +} + +func parseBlockedFlags(t *testing.T, out string) []blockedFlagRow { + t.Helper() + var rows []blockedFlagRow + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rows); err != nil { + t.Fatalf("parse bd blocked --flags --json: %v\n%s", err, out) + } + if rows == nil { + t.Fatalf("bd blocked --flags --json printed null, want an array: %s", out) + } + return rows +} + +// flagStdin is every id the fixture created, in title order, plus an id that +// names no row, a blank line and a repeat: the omitted, ignored and collapsed +// cases. +func flagStdin(f *remoteRolesFixture) string { + titles := make([]string, 0, len(f.ids)) + for title := range f.ids { + titles = append(titles, title) + } + sort.Strings(titles) + var b strings.Builder + for _, title := range titles { + b.WriteString(f.ids[title] + "\n") + } + b.WriteString("\n " + f.ids[titles[0]] + " \ne2e-no-such-row\n") + return b.String() +} + +// TestE2E_BlockedFlagsOverHTTP runs `bd blocked --flags --json` (the blocked +// flag per id read from stdin, the readiness column gc projects) through the +// real `bd` binary against an in-process `bd serve` and against a local +// embedded workspace holding the same graph, and requires the same JSON once +// each generated id is mapped to its title. Over HTTP the read is +// listBlockingAnnotations with is_blocked; an older server's handshake is +// refused before the dial, naming dependencies.blocking.isBlocked. +// +// It also pins that this is the FLAG and not the listing: a row whose stored +// flag is set with no nameable blocker reads true here and is absent from +// `bd blocked --json`. +func TestE2E_BlockedFlagsOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + + seed := func(f *remoteRolesFixture) { + f.seed() + blocker := f.create("blocker", "-t", "task") + gp := f.create("gate-parent", "-t", "epic") + kid := f.create("gate-kid", "-t", "task") + f.mustRun(nil, "dep", "add", kid, gp, "--type", "parent-child") + f.mustRun(nil, "dep", "add", gp, blocker) + } + + t.Run("served equals local", func(t *testing.T) { + addr := startE2EServer(t) + local := newEmbeddedFixture(t, bin) + seed(local) + served := newHTTPFixture(t, bin, addr) + seed(served) + + lr := runBDStdin(t, local, flagStdin(local), "blocked", "--flags", "--json") + sr := runBDStdin(t, served, flagStdin(served), "blocked", "--flags", "--json") + if lr.code != 0 || sr.code != 0 { + t.Fatalf("exit local=%d served=%d\nlocal: %s %s\nserved: %s %s", lr.code, sr.code, lr.stdout, lr.stderr, sr.stdout, sr.stderr) + } + normalize := func(f *remoteRolesFixture, rows []blockedFlagRow) []blockedFlagRow { + titleOf := map[string]string{} + for title, id := range f.ids { + titleOf[id] = title + } + out := make([]blockedFlagRow, 0, len(rows)) + for _, row := range rows { + if title, ok := titleOf[row.ID]; ok { + row.ID = title + } + out = append(out, row) + } + return out + } + want := normalize(local, parseBlockedFlags(t, lr.stdout)) + got := normalize(served, parseBlockedFlags(t, sr.stdout)) + if !reflect.DeepEqual(want, got) { + t.Errorf("served bd blocked --flags --json differs from the local command:\nlocal: %v\nserved: %v", want, got) + } + if len(want) != len(local.ids) { + t.Errorf("local answered %d rows for %d ids (the unknown id omitted, the repeat collapsed): %v", len(want), len(local.ids), want) + } + blocked := map[string]bool{} + for _, row := range got { + blocked[row.ID] = row.IsBlocked + } + for title, wantBlocked := range map[string]bool{ + "loner": true, "child-x": true, "gate-parent": true, "gate-kid": true, + "blocker": false, "child-b": false, "parent": false, + } { + if blocked[title] != wantBlocked { + t.Errorf("%s is_blocked = %v, want %v (rows %v)", title, blocked[title], wantBlocked, got) + } + } + // Byte-for-byte the same members in the same order. + if strings.Count(lr.stdout, "\n") != strings.Count(sr.stdout, "\n") { + t.Errorf("line counts differ:\nlocal:\n%s\nserved:\n%s", lr.stdout, sr.stdout) + } + // Nothing piped is an empty array. + if r := runBDStdin(t, served, "", "blocked", "--flags", "--json"); r.code != 0 || strings.TrimSpace(r.stdout) != "[]" { + t.Errorf("empty stdin: exit %d stdout %q, want [] and exit 0", r.code, r.stdout) + } + // The listing's filters are refused, not silently ignored. + if r := runBDStdin(t, served, "x\n", "blocked", "--flags", "--json", "--label", "a"); r.code != 1 { + t.Errorf("--flags --label exited %d, want 1: %s", r.code, r.stdout) + } + }) + + t.Run("the flag, not the listing", func(t *testing.T) { + addr, reference := startE2EServerWithStore(t, e2eProjectID, e2eDatabase) + served := newHTTPFixture(t, bin, addr) + stale := served.create("stale") + if err := reference.RunEventsMaintenanceTx(context.Background(), func(ctx context.Context, tx issueops.DBTX) error { + _, err := tx.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 WHERE id = ?", stale) + return err + }); err != nil { + t.Fatalf("set the stale flag: %v", err) + } + rows := parseBlockedFlags(t, runBDStdin(t, served, stale+"\n", "blocked", "--flags", "--json").stdout) + if len(rows) != 1 || rows[0].ID != stale || !rows[0].IsBlocked { + t.Errorf("--flags for a stale-flagged row = %v, want [{%s true}]", rows, stale) + } + listing := served.mustRun(nil, "blocked", "--json").stdout + if strings.Contains(listing, stale) { + t.Errorf("bd blocked --json lists %s, which has no nameable blocker: %s", stale, listing) + } + }) + + t.Run("raw is the stored column", func(t *testing.T) { + addr, reference := startE2EServerWithStore(t, e2eProjectID, e2eDatabase) + local := newEmbeddedFixture(t, bin) + seed(local) + served := newHTTPFixture(t, bin, addr) + seed(served) + + flagsOf := func(f *remoteRolesFixture, stdin string, args ...string) map[string]bool { + t.Helper() + r := runBDStdin(t, f, stdin, append([]string{"blocked", "--flags", "--json"}, args...)...) + if r.code != 0 { + t.Fatalf("bd blocked --flags --json %v exited %d: %s %s", args, r.code, r.stdout, r.stderr) + } + titleOf := map[string]string{} + for title, id := range f.ids { + titleOf[id] = title + } + out := map[string]bool{} + for _, row := range parseBlockedFlags(t, r.stdout) { + key := row.ID + if title, ok := titleOf[row.ID]; ok { + key = title + } + out[key] = row.IsBlocked + } + return out + } + + localRaw := flagsOf(local, flagStdin(local), "--raw") + servedRaw := flagsOf(served, flagStdin(served), "--raw") + if !reflect.DeepEqual(localRaw, servedRaw) { + t.Errorf("served --raw differs from local --raw:\nlocal: %v\nserved: %v", localRaw, servedRaw) + } + lr := runBDStdin(t, local, flagStdin(local), "blocked", "--flags", "--json", "--raw") + sr := runBDStdin(t, served, flagStdin(served), "blocked", "--flags", "--json", "--raw") + if strings.Count(lr.stdout, "\n") != strings.Count(sr.stdout, "\n") { + t.Errorf("--raw line counts differ:\nlocal:\n%s\nserved:\n%s", lr.stdout, sr.stdout) + } + + // child-x is blocked ONLY by an unsatisfied external ref: the policy + // flag says blocked, the stored column does not. + decorated := flagsOf(served, flagStdin(served)) + if !decorated["child-x"] || servedRaw["child-x"] || localRaw["child-x"] { + t.Errorf("child-x (externally blocked only): --flags %v, --raw served %v local %v; want true, false, false", + decorated["child-x"], servedRaw["child-x"], localRaw["child-x"]) + } + // Locally blocked rows read the same either way. + for _, title := range []string{"loner", "gate-parent", "gate-kid", "blocker"} { + if decorated[title] != servedRaw[title] { + t.Errorf("%s: --flags %v, --raw %v; a local blocker is the column", title, decorated[title], servedRaw[title]) + } + } + + // A stale flag (set, no blocker) is the column, so both read true. + stale := served.create("stale") + if err := reference.RunEventsMaintenanceTx(context.Background(), func(ctx context.Context, tx issueops.DBTX) error { + _, err := tx.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 WHERE id = ?", stale) + return err + }); err != nil { + t.Fatalf("set the stale flag: %v", err) + } + if got := flagsOf(served, stale+"\n", "--raw"); !got["stale"] { + t.Errorf("--raw for the stale-flagged row = %v, want true", got) + } + if got := flagsOf(served, stale+"\n"); !got["stale"] { + t.Errorf("--flags for the stale-flagged row = %v, want true", got) + } + + // --raw qualifies --flags only. + if r := runBDStdin(t, served, "", "blocked", "--json", "--raw"); r.code != 1 || !strings.Contains(r.stdout+r.stderr, "--raw") { + t.Errorf("bd blocked --raw without --flags: exit %d %s %s, want exit 1 naming --raw", r.code, r.stdout, r.stderr) + } + + // A server that applies the policy answers the decorated flag and the + // wire cannot ask it for the column: --raw is refused there, before + // the annotation read, rather than answered with the policy flag. + proxyAddr, counter := startMaskingProxy(t, addr, func(body map[string]any) { + caps, _ := body["capabilities"].([]any) + body["capabilities"] = append(caps, "policy.external_dependencies") + }) + enforced := newHTTPFixture(t, bin, proxyAddr) + counter.reset() + r := runBDStdin(t, enforced, served.ids["child-x"]+"\n", "blocked", "--flags", "--json", "--raw") + if r.code != 1 || !strings.Contains(r.stdout+r.stderr, "policy.external_dependencies") { + t.Errorf("--raw against a policy server: exit %d %s %s, want exit 1 naming the policy token", r.code, r.stdout, r.stderr) + } + for _, p := range counter.snapshot() { + if strings.Contains(p, "/v0/beads/dependencies/blocking") { + t.Errorf("a refused --raw dialed %s", p) + } + } + }) + + t.Run("older server mask", func(t *testing.T) { + addr := startE2EServer(t) + proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) + f := newHTTPFixture(t, bin, proxyAddr) + counter.reset() + r := runBDStdin(t, f, "e2e-anything\n", "blocked", "--flags", "--json") + if r.code != 1 { + t.Fatalf("bd blocked --flags against a server without the token exited %d, want 1: %s", r.code, r.stdout) + } + if !strings.Contains(r.stdout+r.stderr, "dependencies.blocking.isBlocked") { + t.Errorf("refusal does not name the missing capability: stdout=%s stderr=%s", r.stdout, r.stderr) + } + for _, p := range counter.snapshot() { + if strings.Contains(p, "/v0/beads/dependencies/blocking") { + t.Errorf("a refused bd blocked --flags dialed %s", p) + } + } + }) +} diff --git a/backend/http/e2e_blocked_test.go b/backend/http/e2e_blocked_test.go new file mode 100644 index 000000000..786379fba --- /dev/null +++ b/backend/http/e2e_blocked_test.go @@ -0,0 +1,229 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "reflect" + "slices" + "strings" + "testing" +) + +// TestE2E_BlockedOverHTTP runs `bd blocked --json` (the blocked door gc's +// readiness projection reads) through the real `bd` binary against an +// in-process `bd serve`, and against a local embedded workspace holding the +// same graph. +// +// Over HTTP the CLI takes the store's BlockedLister, which dials +// GET /v0/beads/blocked behind blocked.list. The two answers must be the same +// JSON once the values that cannot agree between two workspaces (each row's +// generated id, and its timestamps) are mapped out: the same rows, the same +// members, the same blockers (local, parent-attributed and external), in the +// same order. Against a server that does not advertise blocked.list (an older +// server's handshake) the client refuses before the dial, naming the token. +func TestE2E_BlockedOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + + seedBlocked := func(f *remoteRolesFixture) { + f.seed() + // A plain blocks edge, a blocked parent's child attributed to the + // parent, and a labelled blocked row for the filter arms. + blocker := f.create("blocker", "-t", "task", "-p", "1") + gp := f.create("gate-parent", "-t", "epic", "-p", "2") + // Its own priority: rows tied on priority and created_at fall to the + // id order, and ids are generated per workspace, so a tie would order + // the two workspaces differently. + kid := f.create("gate-kid", "-t", "task", "-p", "3") + tagged := f.create("tagged", "-t", "task", "-p", "1", "-l", "lane:a") + // Blocked by nothing but an unsatisfied external ref, as child-x is, + // and labelled: the decorator lists such rows itself, after the label + // query ran, so the filter arms must judge them too. + extTagged := f.create("ext-tagged", "-t", "task", "-p", "4", "-l", "lane:a") + f.mustRun(nil, "dep", "add", kid, gp, "--type", "parent-child") + f.mustRun(nil, "dep", "add", gp, blocker) + f.mustRun(nil, "dep", "add", tagged, blocker) + f.mustRun(nil, "dep", "add", extTagged, "external:other:cap") + } + + local := newEmbeddedFixture(t, bin) + seedBlocked(local) + served := newHTTPFixture(t, bin, addr) + seedBlocked(served) + + for _, args := range [][]string{ + {"blocked", "--json"}, + {"blocked", "--json", "--label", "lane:a"}, + {"blocked", "--json", "--label-any", "lane:a"}, + {"blocked", "--json", "--exclude-label", "lane:a"}, + {"blocked", "--json", "--parent", "{gate-parent}"}, + {"blocked", "--json", "--label", "no-such-label"}, + } { + t.Run(strings.Join(args[1:], " "), func(t *testing.T) { + want := local.mustRun(nil, local.expand(args)...).stdout + got := served.mustRun(nil, served.expand(args)...).stdout + if strings.TrimSpace(want) == "null" || strings.TrimSpace(got) == "null" { + t.Fatalf("bd blocked --json printed null: local=%s served=%s", want, got) + } + if w, g := normalizeBlocked(t, local, want), normalizeBlocked(t, served, got); !reflect.DeepEqual(w, g) { + t.Errorf("served bd blocked --json differs from the local command:\nlocal: %v\nserved: %v", w, g) + } + if w, g := blockedKeyOrder(t, want), blockedKeyOrder(t, got); !reflect.DeepEqual(w, g) { + t.Errorf("served member order %v differs from the local command's %v", g, w) + } + }) + } + + t.Run("names the external blocker", func(t *testing.T) { + got := normalizeBlocked(t, served, served.mustRun(nil, "blocked", "--json").stdout) + found := false + for _, row := range got { + if row["id"] == "child-x" { + found = reflect.DeepEqual(row["blocked_by"], []any{"external:other:cap"}) + } + } + if !found { + t.Errorf("served bd blocked --json does not list child-x blocked by its external ref: %v", got) + } + }) + + // The arms above compare the two answers, which agree just as well when + // both list a row the filter rules out. These pin which rows each filter + // leaves, the external-only ones (child-x unlabelled, ext-tagged + // labelled) among them. + for _, tc := range []struct { + args []string + want []string // the whole listing, when set + listed, ruled []string + }{ + {args: []string{"--label", "lane:a"}, want: []string{"ext-tagged", "tagged"}}, + {args: []string{"--label-any", "lane:a"}, want: []string{"ext-tagged", "tagged"}}, + {args: []string{"--label", "no-such-label"}, want: []string{}}, + {args: []string{"--exclude-label", "lane:a"}, listed: []string{"child-x"}, ruled: []string{"ext-tagged", "tagged"}}, + } { + t.Run("filters external-only rows "+strings.Join(tc.args, " "), func(t *testing.T) { + for _, f := range []struct { + name string + f *remoteRolesFixture + }{{"local", local}, {"served", served}} { + got := f.f.titlesOf(f.f.mustRun(nil, append([]string{"blocked", "--json"}, tc.args...)...).stdout) + if tc.want != nil && !slices.Equal(got, tc.want) { + t.Errorf("%s bd blocked %s listed %v, want %v", f.name, strings.Join(tc.args, " "), got, tc.want) + } + for _, title := range tc.listed { + if !slices.Contains(got, title) { + t.Errorf("%s bd blocked %s does not list %s: %v", f.name, strings.Join(tc.args, " "), title, got) + } + } + for _, title := range tc.ruled { + if slices.Contains(got, title) { + t.Errorf("%s bd blocked %s lists %s, which the filter rules out: %v", f.name, strings.Join(tc.args, " "), title, got) + } + } + } + }) + } + + t.Run("older server mask", func(t *testing.T) { + proxyAddr, counter := startMaskingProxy(t, addr, mcContextMask) + f := newHTTPFixture(t, bin, proxyAddr) + counter.reset() + r := f.run(nil, "blocked", "--json") + if r.code != 1 { + t.Fatalf("bd blocked against a server without blocked.list exited %d, want 1: %s", r.code, r.stdout) + } + if !strings.Contains(r.stdout+r.stderr, "blocked.list") { + t.Errorf("refusal does not name the missing capability: stdout=%s stderr=%s", r.stdout, r.stderr) + } + for _, p := range counter.snapshot() { + if strings.Contains(p, "/v0/beads/blocked") { + t.Errorf("a refused bd blocked dialed %s; the refusal must come before the dial", p) + } + } + }) +} + +// normalizeBlocked parses `bd blocked --json` and maps out what two +// workspaces cannot share: each id (and each blocker id) becomes the title the +// fixture created it under, and every timestamp member is blanked. +func normalizeBlocked(t *testing.T, f *remoteRolesFixture, out string) []map[string]any { + t.Helper() + var rows []map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rows); err != nil { + t.Fatalf("parse bd blocked --json: %v\n%s", err, out) + } + titleOf := map[string]string{} + for title, id := range f.ids { + titleOf[id] = title + } + var walk func(key string, v any) any + walk = func(key string, v any) any { + switch v := v.(type) { + case map[string]any: + for k, inner := range v { + v[k] = walk(k, inner) + } + return v + case []any: + for i := range v { + v[i] = walk("", v[i]) + } + return v + case string: + if strings.HasSuffix(key, "_at") { + return "&"'`}, want: []byte(`{"s":"&\"'"}`), }, - ) - for _, tc := range cases { + } { t.Run(tc.name, func(t *testing.T) { t.Parallel() got, err := canonicalDurableState(tc.state) @@ -118,9 +159,23 @@ func TestCanonicalDurableStateIsJCS(t *testing.T) { "id": "bd-1", "priority": json.Number("1.0"), "metadata": map[string]any{ - "weight": json.Number("1.50"), - "huge": json.Number("1E300"), - "ordinal": json.Number("9007199254740993"), // 2^53 + 1 + "weight": json.Number("1.50"), + // 1E-300, not 1E300: this test is about number FORM + // canonicalization (ES6 exponential notation), not the admission + // gate. 1E300 denotes an INTEGER value (10^300) past what + // binary64 holds exactly, so refuseUnrepresentableIntegers now + // refuses it -- see version_history_integer_admission_test.go. + // 1E-300 exercises the same exponential FORM without being an + // integer at all, so it stays in scope for this test. + "huge": json.Number("1E-300"), + // 2^53-1, the largest integer binary64 holds exactly. This was + // 2^53+1 until the admission gate began refusing literals that + // binary64 cannot hold: JCS would round such a literal to a + // DIFFERENT VALUE, collapsing two distinct states onto one + // durable_state. That case now lives in + // version_history_integer_admission_test.go, which asserts the + // refusal; this test stays about number FORM canonicalization. + "ordinal": json.Number("9007199254740991"), "tags": []any{json.Number("2.0"), "b", nil}, }, "raw": json.RawMessage(`{ "z" : 10.0e0 , "a" : true }`), @@ -132,9 +187,10 @@ func TestCanonicalDurableStateIsJCS(t *testing.T) { } // RFC 8785: keys sorted, no whitespace, numbers in ES6 Number::toString - // form (1.0 -> 1, 1.50 -> 1.5, 1E300 -> 1e+300, 2^53+1 -> the double it - // rounds to, 10.0e0 -> 10). - want := []byte(`{"id":"bd-1","metadata":{"huge":1e+300,"ordinal":9007199254740992,"tags":[2,"b",null],"weight":1.5},"priority":1,"raw":{"a":true,"z":10}}`) + // form (1.0 -> 1, 1.50 -> 1.5, 1E-300 -> 1e-300, 10.0e0 -> 10). An + // integer-valued literal past 2^53-1, any spelling, never reaches JCS now + // -- the admission gate refuses it first. + want := []byte(`{"id":"bd-1","metadata":{"huge":1e-300,"ordinal":9007199254740991,"tags":[2,"b",null],"weight":1.5},"priority":1,"raw":{"a":true,"z":10}}`) if !bytes.Equal(got, want) { t.Fatalf("canonicalDurableState =\n %s\nwant\n %s", got, want) } @@ -148,7 +204,7 @@ func TestCanonicalDurableStateIsJCS(t *testing.T) { if bytes.Equal(plain, got) { t.Fatalf("plain json.Marshal already equals the canonical form (%s); this test no longer proves canonicalization matters", plain) } - for _, nonCanonical := range []string{"1.0", "1.50", "1E300", "9007199254740993", "10.0e0"} { + for _, nonCanonical := range []string{"1.0", "1.50", "1E-300", "10.0e0"} { if !bytes.Contains(plain, []byte(nonCanonical)) { t.Errorf("plain marshal lost the non-canonical form %q, so the fixture no longer exercises it: %s", nonCanonical, plain) } diff --git a/internal/storage/issueops/version_history_integer_admission_test.go b/internal/storage/issueops/version_history_integer_admission_test.go new file mode 100644 index 000000000..9966afa88 --- /dev/null +++ b/internal/storage/issueops/version_history_integer_admission_test.go @@ -0,0 +1,186 @@ +package issueops + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +func issueWithMetadata(meta string) types.Issue { + return types.Issue{ID: "be-1", Title: "t", Metadata: json.RawMessage(meta)} +} + +// A metadata integer past 2^53 used to be ROUNDED by jcs.Transform, which +// collapsed two distinct issue states onto one durable_state and therefore +// one content token. It must be refused instead, so the loss is loud. +func TestCanonicalDurableStateRefusesIntegerPastBinary64(t *testing.T) { + _, err := canonicalDurableState(issueWithMetadata(`{"n":9007199254740993}`)) + if err == nil { + t.Fatal("canonicalDurableState(metadata 9007199254740993) = nil error, want a refusal") + } + if !errors.Is(err, ErrIntegerNotRepresentable) { + t.Fatalf("error = %v, want ErrIntegerNotRepresentable", err) + } +} + +// The defect this admission gate exists to prevent, stated directly: two +// DISTINCT states must never render to the same bytes. +func TestCanonicalDurableStateNeverCollapsesTwoDistinctStates(t *testing.T) { + // Before the admission gate, BOTH of these canonicalized, and both to + // the same bytes: 2^53+1 rounded down onto 2^53. One content token for + // two live states. Now the unrepresentable side is refused, so the pair + // can never be confused for one another. + a, errA := canonicalDurableState(issueWithMetadata(`{"n":9007199254740993}`)) + b, errB := canonicalDurableState(issueWithMetadata(`{"n":9007199254740991}`)) + if errA == nil && errB == nil && string(a) == string(b) { + t.Fatalf("two distinct states produced identical durable_state %s", a) + } + if errA == nil { + t.Fatalf("the unrepresentable state canonicalized instead of being refused: %s", a) + } + if errB != nil { + t.Fatalf("the representable state was refused: %v", errB) + } +} + +// The bound itself, both sides of it, bare-integer spelling. +func TestCanonicalDurableStateIntegerBoundary(t *testing.T) { + for _, tc := range []struct { + name string + meta string + refused bool + }{ + {"2^53-1 accepted", `{"n":9007199254740991}`, false}, + {"2^53 refused", `{"n":9007199254740992}`, true}, + {"2^53+1 refused", `{"n":9007199254740993}`, true}, + {"negative past bound refused", `{"n":-9007199254740993}`, true}, + {"beyond int64 refused", `{"n":18446744073709551616}`, true}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := canonicalDurableState(issueWithMetadata(tc.meta)) + if tc.refused && err == nil { + t.Fatalf("%s: nil error, want refusal", tc.meta) + } + if !tc.refused && err != nil { + t.Fatalf("%s: %v, want acceptance", tc.meta, err) + } + }) + } +} + +// Ordinary fractions must keep working: they are not exactly representable +// either, but ES6 shortest-round-trip returns them to the same binary64 +// value, so they neither round to a different value nor collide. +// +// 1e300 is deliberately NOT in this list: it is a mathematical integer +// (10^300) whose magnitude vastly exceeds 2^53-1, so it is refused like any +// other unrepresentable integer, regardless of spelling. See +// TestCanonicalDurableStateRefusesHugeExponentInteger below. +func TestCanonicalDurableStateStillAcceptsOrdinaryDecimals(t *testing.T) { + for _, meta := range []string{`{"n":0.1}`, `{"n":1.0}`, `{"n":-2.5}`, `{"n":0}`} { + if _, err := canonicalDurableState(issueWithMetadata(meta)); err != nil { + t.Fatalf("canonicalDurableState(%s) = %v, want acceptance", meta, err) + } + } +} + +// 1e300 denotes the integer 10^300 at a magnitude vastly past 2^53-1, and +// is subject to the same collision defect this gate +// exists to close -- binary64 spacing near 10^300 is roughly 2^248, so a wide +// range of distinct large integers near 10^300 collapse onto that one +// double, same failure mode as the 2^53 boundary, larger window. The +// spec for this predicate is "value not form, any spelling, no exceptions" +// (see version_history.go's refuseUnrepresentableIntegers doc comment), +// applied literally that refuses 1e300 -- but two pre-gate tests assumed it +// stayed accepted, one of which lived right here until this admission gate +// (see version_history_durable_state_test.go's "huge" fixture, moved off +// 1E300 for the same reason). +// +// RESOLVED: REFUSE 1e300. The rule applies +// literally, with no notation carve-out -- admitting a spelling this large +// while refusing 9007199254740993 would be indefensible, since the collision +// window near 10^300 (binary64 spacing ~2^248) is vastly larger than the one +// at the 2^53 boundary this gate was built to close. This is final, not +// provisional. +func TestCanonicalDurableStateRefusesHugeExponentInteger(t *testing.T) { + _, err := canonicalDurableState(issueWithMetadata(`{"n":1e300}`)) + if err == nil { + t.Fatal("canonicalDurableState(metadata 1e300) = nil error, want a refusal") + } + if !errors.Is(err, ErrIntegerNotRepresentable) { + t.Fatalf("error = %v, want ErrIntegerNotRepresentable", err) + } +} + +// THE GAP QA FOUND IN THE FIRST CUT OF THIS GATE. The first patch's +// refuseUnrepresentableIntegers skipped any literal containing '.', 'e', or +// 'E' outright, reasoning that non-integer literals round-trip losslessly +// through ES6 shortest-round-trip. That reasoning covers genuinely +// fractional literals (0.1, 1.5) but not integer-VALUED literals written in +// decimal or exponential FORM: 9007199254740993.0, 9.007199254740993e15, and +// 90071992547409930e-1 all denote the same integer VALUE as the bare literal +// 9007199254740993, and all three walked straight past the first gate and +// canonicalized to the identical colliding target 9007199254740992 -- the +// exact defect class this gate exists to close, just reachable through +// spelling instead of magnitude. +// +// The predicate under test must inspect the VALUE (the binary64 nearest +// to the literal) rather than the literal's spelling, so it must +// refuse and accept the same values regardless of which of the three JSON +// number forms (integer, decimal, exponent) carries them. This is pinned +// here for the bound itself; ordinary small decimals are covered separately +// by TestCanonicalDurableStateStillAcceptsOrdinaryDecimals. +func TestCanonicalDurableStateIntegerBoundaryAllSpellings(t *testing.T) { + for _, tc := range []struct { + name string + lit string + refused bool + }{ + // at the bound (2^53-1 = 9007199254740991): every spelling accepted. + {"at bound, bare", `9007199254740991`, false}, + {"at bound, decimal", `9007199254740991.0`, false}, + {"at bound, exponent", `9.007199254740991e15`, false}, + + // one past the bound (2^53+1 = 9007199254740993): every spelling + // refused. The decimal and exponent rows are the QA-found gap. + {"past bound, bare", `9007199254740993`, true}, + {"past bound, decimal (THE GAP)", `9007199254740993.0`, true}, + {"past bound, exponent (THE GAP)", `9.007199254740993e15`, true}, + {"past bound, exponent negative-exp form (THE GAP)", `90071992547409930e-1`, true}, + } { + t.Run(tc.name, func(t *testing.T) { + meta := `{"n":` + tc.lit + `}` + _, err := canonicalDurableState(issueWithMetadata(meta)) + if tc.refused && err == nil { + t.Fatalf("metadata %s: nil error, want refusal", meta) + } + if !tc.refused && err != nil { + t.Fatalf("metadata %s: %v, want acceptance", meta, err) + } + if tc.refused && !errors.Is(err, ErrIntegerNotRepresentable) { + t.Fatalf("metadata %s: error = %v, want ErrIntegerNotRepresentable", meta, err) + } + }) + } +} + +// Same property as TestCanonicalDurableStateNeverCollapsesTwoDistinctStates, +// spelled in decimal form -- the exact QA-found gap stated as a distinctness +// property rather than a table row: two states differing only in a +// decimal-spelled metadata integer past 2^53 must never render to the same +// bytes. +func TestCanonicalDurableStateNeverCollapsesDecimalSpellingOfDistinctStates(t *testing.T) { + a, errA := canonicalDurableState(issueWithMetadata(`{"n":9007199254740993.0}`)) + b, errB := canonicalDurableState(issueWithMetadata(`{"n":9007199254740991.0}`)) + if errA == nil && errB == nil && string(a) == string(b) { + t.Fatalf("two distinct states produced identical durable_state %s", a) + } + if errA == nil { + t.Fatalf("the unrepresentable state (9007199254740993.0) canonicalized instead of being refused: %s", a) + } + if errB != nil { + t.Fatalf("the representable state (9007199254740991.0) was refused: %v", errB) + } +} diff --git a/internal/storage/issueops/version_history_number_admission_test.go b/internal/storage/issueops/version_history_number_admission_test.go new file mode 100644 index 000000000..f68193aa9 --- /dev/null +++ b/internal/storage/issueops/version_history_number_admission_test.go @@ -0,0 +1,298 @@ +package issueops + +import ( + "errors" + "math" + "math/rand" + "runtime" + "strconv" + "strings" + "testing" +) + +// admitNumber runs the admission gate over one JSON number literal in the +// position the mint meets it: an object member of an issue's metadata. +func admitNumber(lit string) error { + return refuseUnrepresentableIntegers([]byte(`{"n":` + lit + `}`)) +} + +// The admission rule, stated once for the tests below: a JSON number literal +// is admitted into a durable_state if and only if the binary64 nearest to it +// has magnitude at most 2^53-1, whatever the literal's spelling (integer, +// decimal or exponent), and a literal that overflows binary64 is refused. +// Ordinary fractions such as 0.1 stay admitted, and so do magnitudes too small +// for binary64, which round to zero. + +// A fraction whose nearest double lies at or beyond 2^53 used to be admitted: +// the old rule looked only at literals that denote an integer VALUE, and +// 9007199254740993.5 is not one. jcs.Transform then rounds it to the integer +// 9007199254740994, so two distinct issue states shared one durable_state, +// which is exactly what the gate exists to prevent. Refusing must depend on +// where the nearest double falls, not on whether the literal looks integral. +func TestNumberAdmissionRefusesFractionsWhoseNearestDoubleIsPastTheBound(t *testing.T) { + for _, tc := range []struct { + lit string + refused bool + why string + }{ + {"9007199254740993.5", true, "nearest double is 2^53+2"}, + {"9007199254740994.5", true, "nearest double is 2^53+2"}, + {"9007199254740991.5", true, "a tie between 2^53-1 and 2^53 that rounds to the even 2^53"}, + {"9007199254740991.75", true, "nearest double is 2^53"}, + {"-9007199254740993.5", true, "magnitude decides, not sign"}, + {"9007199254740990.5", false, "a tie that rounds to the even 2^53-2, inside the range"}, + {"9007199254740991.25", false, "nearest double is 2^53-1, inside the range"}, + {"9007199254740991.0", false, "exactly 2^53-1"}, + } { + t.Run(tc.lit, func(t *testing.T) { + err := admitNumber(tc.lit) + if tc.refused { + if !errors.Is(err, ErrIntegerNotRepresentable) { + t.Fatalf("admitNumber(%s) = %v, want ErrIntegerNotRepresentable (%s)", tc.lit, err, tc.why) + } + if _, mintErr := canonicalDurableState(issueWithMetadata(`{"n":` + tc.lit + `}`)); !errors.Is(mintErr, ErrIntegerNotRepresentable) { + t.Fatalf("canonicalDurableState(metadata %s) = %v, want ErrIntegerNotRepresentable", tc.lit, mintErr) + } + return + } + if err != nil { + t.Fatalf("admitNumber(%s) = %v, want acceptance (%s)", tc.lit, err, tc.why) + } + }) + } +} + +// numberAdmissionCorpus is every literal the closure property is checked +// over: the boundary cases the rule was written around, every spelling of the +// values on either side of 2^53, and pseudo-random doubles in three renderings. +// The seed is fixed so a failure names a literal that reproduces. +func numberAdmissionCorpus() []string { + corpus := []string{ + "0", "-0", "0.0", "-0.0", "1", "1.0", "-2.5", "0.1", "3.14159265358979323846", "0.30000000000000004", + "5e-324", "4.9e-324", "2.2250738585072014e-308", "1e-7", "1e-400", "1e-100000000", + "9007199254740991", "9007199254740992", "9007199254740993", "9.007199254740991e15", "90071992547409930e-1", + "9007199254740993.5", "9007199254740994.5", "9007199254740991.5", "9007199254740990.5", + "1727000000000000000", "1727000000000000123", "1e21", "1e300", "1.7976931348623157e308", + "123456789012345678901234567890", "18446744073709551616", + } + // Every spelling of the integers and near-integers either side of the bound. + for delta := int64(-6); delta <= 6; delta++ { + base := strconv.FormatInt(int64(1)<<53+delta, 10) + for _, frac := range []string{"", ".0", ".25", ".5", ".75", ".999999"} { + corpus = append(corpus, base+frac, "-"+base+frac) + } + } + rng := rand.New(rand.NewSource(20260930)) + for _, scale := range []float64{1, 1e3, 1e9, 1e15, 1e-9} { + for range 60 { + f := (rng.Float64()*2 - 1) * scale + corpus = append(corpus, + strconv.FormatFloat(f, 'g', -1, 64), + strconv.FormatFloat(f, 'e', 17, 64), + strconv.FormatFloat(f, 'f', -1, 64)) + } + } + for range 200 { + f := math.Float64frombits(rng.Uint64()) + if math.IsNaN(f) || math.IsInf(f, 0) { + continue + } + corpus = append(corpus, strconv.FormatFloat(f, 'g', -1, 64), strconv.FormatFloat(f, 'e', -1, 64)) + } + return corpus +} + +// The gate must be closed under its own canonicalization: whatever it admits +// canonicalizes to a form it also admits. Otherwise a value can pass at the +// door and become unmintable one step later, which is what 9007199254740991.5 +// did (it rounds to 2^53, and the canonical bytes are then refused). Checked as +// a property over a corpus, not by example. +func TestNumberAdmissionIsClosedUnderCanonicalization(t *testing.T) { + admitted := 0 + for _, lit := range numberAdmissionCorpus() { + doc := []byte(`{"n":` + lit + `}`) + if err := refuseUnrepresentableIntegers(doc); err != nil { + continue // refused literals are outside the property + } + admitted++ + canonical, err := jcsCanonicalize(doc) + if err != nil { + t.Errorf("literal %s: jcsCanonicalize: %v", lit, err) + continue + } + if err := refuseUnrepresentableIntegers(canonical); err != nil { + t.Errorf("literal %s is admitted but its canonical form %s is refused: %v", lit, canonical, err) + } + } + if admitted < 100 { + t.Fatalf("only %d corpus literals were admitted; the property is close to vacuous", admitted) + } +} + +// The classification must not depend on how large an exponent the literal +// spells. The old rule parsed every literal as an exact rational, so a huge +// exponent either cost a million-digit integer or fell outside what the +// parser accepts and came back as an invalid literal, a different class of +// error from the refusal it is. Overflow is a refusal; a magnitude too small +// for binary64 is admitted. +func TestNumberAdmissionClassifiesByMagnitudeWhateverTheExponent(t *testing.T) { + for _, tc := range []struct { + lit string + refused bool + }{ + {"1e100000000", true}, + {"1e1000000", true}, + {"1e309", true}, + {"-1e400", true}, + {"1.7976931348623159e308", true}, + {"1e-100000000", false}, + {"1e-400", false}, + {"-1e-400", false}, + {"5e-324", false}, + } { + t.Run(tc.lit, func(t *testing.T) { + err := admitNumber(tc.lit) + switch { + case tc.refused && !errors.Is(err, ErrIntegerNotRepresentable): + t.Fatalf("admitNumber(%s) = %v, want ErrIntegerNotRepresentable", tc.lit, err) + case !tc.refused && err != nil: + t.Fatalf("admitNumber(%s) = %v, want acceptance", tc.lit, err) + } + }) + } +} + +// The gate allocates nothing proportional to an exponent. Measured as an +// allocation count and never as wall time, so a loaded machine cannot flake it, +// and it is what distinguishes a classifier that builds a million-digit integer +// from one that does not. +// +// The count is exact only without the race detector. Under it sync.Pool drops +// one Put in four at random (sync/pool.go), and the refusal builds its error with +// fmt.Errorf, whose printer comes from a pool, so a call now and then allocates a +// printer the pool would have supplied: a literal that costs 22 allocations reads +// 23, or 24, on a run where a printer was dropped, and comparing two +// independently averaged samples with a strict > is a coin flip. That noise only +// ever ADDS allocations, so the fewest allocations over many single-call samples +// is the count without it, and it is the same on every run. The comparison stays +// strict, with no margin to tune, and it loses no sensitivity: a classifier that +// builds per-exponent state does so on every call, so it raises the fewest too. +func TestNumberAdmissionAllocationsDoNotGrowWithTheExponent(t *testing.T) { + // The chance that every one of these samples is noisy is under one in 10^19. + const samples = 32 + fewest := func(lit string) float64 { + doc := []byte(`{"n":` + lit + `}`) + least := math.Inf(1) + for range samples { + least = math.Min(least, testing.AllocsPerRun(1, func() { _ = refuseUnrepresentableIntegers(doc) })) + } + return least + } + baseline := fewest("1e400") + for _, lit := range []string{"1e1000000", "1e100000000", "-1e1000000"} { + if got := fewest(lit); got > baseline { + t.Errorf("%s allocates %v times per call against %v for 1e400: allocation grows with the exponent", lit, got, baseline) + } + } +} + +// TestNumberAdmissionBytesDoNotGrowWithTheExponent is the count bound's twin, and finding 6 +// of the #6661 review. The count bound passes a classifier that builds the number it is +// asked about: one big-integer allocation is one allocation whether it holds 400 bits or a +// hundred million. What grows with the exponent is then the size of the allocation, so this +// bounds bytes: the fewest bytes one call allocates for a literal with a huge exponent is +// no more than for a small one, plus a margin and a few bytes for each byte of literal (a +// refusal names the literal in its message). +// +// The fewest over many single-call samples, as in the count bound and for the same reason: +// whatever else allocates while a sample runs only ever ADDS bytes, so the minimum is the +// cost of the call itself. TotalAlloc is read before and after each call, with GOMAXPROCS +// pinned to one for the measurement as testing.AllocsPerRun does, so a background +// goroutine does not add to a sample. +func TestNumberAdmissionBytesDoNotGrowWithTheExponent(t *testing.T) { + const ( + samples = 32 + smallLiteral = "1e400" + // What the bound allows beyond the small literal's bytes: a margin, and a few + // bytes for each byte a longer literal adds to the refusal's message. + slackBytes = 1024 + perLitByte = 16 + ) + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + fewest := func(lit string) uint64 { + doc := []byte(`{"n":` + lit + `}`) + least := uint64(math.MaxUint64) + var before, after runtime.MemStats + for range samples { + runtime.ReadMemStats(&before) + _ = refuseUnrepresentableIntegers(doc) + runtime.ReadMemStats(&after) + least = min(least, after.TotalAlloc-before.TotalAlloc) + } + return least + } + baseline := fewest(smallLiteral) + for _, lit := range []string{"1e1000000", "1e100000000", "-1e1000000"} { + limit := baseline + slackBytes + perLitByte*uint64(max(len(lit)-len(smallLiteral), 0)) + if got := fewest(lit); got > limit { + t.Errorf("%s allocates %d bytes per call against %d for %s (limit %d): bytes grow with the exponent", lit, got, baseline, smallLiteral, limit) + } + } +} + +// The sentinel names the range it enforces. Its old wording said a literal +// was not exactly representable, which no longer describes what is refused: +// 9007199254740993.5 is a fraction, and the rule is about magnitude. The +// identifier keeps its historical name so nothing that matches it with +// errors.Is has to change; the text is what a person reads. +func TestIntegerNotRepresentableTextNamesTheIJSONRange(t *testing.T) { + const want = "number is outside the I-JSON exact-integer range (magnitude above 2^53-1)" + got := ErrIntegerNotRepresentable.Error() + if got != want { + t.Fatalf("ErrIntegerNotRepresentable.Error() = %q, want %q", got, want) + } + if !strings.Contains(got, "I-JSON") { + t.Errorf("sentinel text %q does not name I-JSON", got) + } + if strings.Contains(got, "cannot represent") { + t.Errorf("sentinel text %q still uses the old wording", got) + } + err := admitNumber("9007199254740993") + if err == nil || err.Error() != want+": 9007199254740993" { + t.Fatalf("refusal text = %v, want the sentinel followed by the offending literal", err) + } +} + +// Nothing the store already accepted becomes admitted or refused differently. +// Dolt does not hand a literal back the way it was written: an integer up to +// int64 is stored exactly and every other number is stored as a double, so what +// the mint sees is the right-hand column. Each row pins the verdict on that +// column, which is what production meets. The verdicts are the ones the rule +// had before it changed, so this stays green through the change and fails only +// if the new rule moves one. +func TestNumberAdmissionKeepsTheVerdictOnEveryLiteralAsTheStoreReturnsIt(t *testing.T) { + for _, tc := range []struct { + written string + returned string // what a Dolt JSON column hands back for it + refused bool + }{ + {"9007199254740993", "9007199254740993", true}, + {"1727000000000000123", "1727000000000000123", true}, + {"9007199254740993.5", "9007199254740994", true}, + {"1.0", "1", false}, + {"0.1", "0.1", false}, + {"3.14159265358979323846", "3.141592653589793", false}, + {"1e300", "1" + strings.Repeat("0", 300), true}, + {"123456789012345678901234567890", "123456789012345680000000000000", true}, + } { + t.Run(tc.written, func(t *testing.T) { + err := admitNumber(tc.returned) + if tc.refused && !errors.Is(err, ErrIntegerNotRepresentable) { + t.Fatalf("admitNumber(%s), the store's rendering of %s = %v, want a refusal", tc.returned, tc.written, err) + } + if !tc.refused && err != nil { + t.Fatalf("admitNumber(%s), the store's rendering of %s = %v, want acceptance", tc.returned, tc.written, err) + } + }) + } +} diff --git a/internal/storage/issueops/version_history_preenable_scan_test.go b/internal/storage/issueops/version_history_preenable_scan_test.go new file mode 100644 index 000000000..74f57b903 --- /dev/null +++ b/internal/storage/issueops/version_history_preenable_scan_test.go @@ -0,0 +1,680 @@ +package issueops + +import ( + "context" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "reflect" + "regexp" + "sort" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage/sqlbuild" + "github.com/steveyegge/beads/internal/types" +) + +// With versioned history on, the mint runs in the same transaction as the +// mutation that reaches it, so refusing at the mint IS refusing the write: a +// number outside the I-JSON range aborts the writer's own transaction and +// nothing it did is committed. The mint reads the whole issue, so the number +// can be in any field that holds one: in metadata, or in a gate's timeout, +// which encoding/json writes as a count of nanoseconds. The mint's reads run +// first and its two writes, the version row and the current_revision advance, +// come last, so a refusal must arrive after the reads and before either write. +// +// The mock scripts exactly the reads. It expects no INSERT into issue_versions +// and no UPDATE of issues.current_revision, so a mint that wrote before it +// refused would make the mock fail the call with an unexpected-statement error +// instead of the refusal asserted here. The caller's own rollback closes the +// transaction, as every store does when the mint returns an error. +// +// The refusal names the top-level field the number was found in, as well as the +// literal, so a person who reads it knows which part of the issue to fix. +func TestRecordVersionInTxRefusesANumberOutsideTheRangeAndWritesNothing(t *testing.T) { + for _, tc := range []struct { + name string + column string + value driver.Value + literal string + field string + }{ + {"a number in metadata", "metadata", `{"ts":1727000000000000000}`, "1727000000000000000", "metadata"}, + {"a gate timeout past 2^53-1 nanoseconds", "timeout_ns", int64(2600 * time.Hour), "9360000000000000", "timeout"}, + } { + t.Run(tc.name, func(t *testing.T) { + const id = "bd-9" + _, mock, tx := beginMockTx(t) + defer ScopeVersionedHistoryTransaction(tx, true)() + + values := issueRowValues(id, "carries an out-of-range number") + for i, col := range issueColumns() { + if col == tc.column { + values[i] = tc.value + } + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT " + IssueSelectColumns + " FROM issues " + sqlbuild.LeaseJoin("issues") + " WHERE id = ?")). + WithArgs(id). + WillReturnRows(issueRows().AddRow(values...)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT label FROM labels WHERE issue_id = ? ORDER BY label")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"label"})) + // design §16.2b's write fence reads participation_generation right after the + // snapshot. A non-NULL value is a participating row, so the mint goes on to the + // refusal this test pins; a NULL one would be a legacy row and mint nothing. + mock.ExpectQuery(regexp.QuoteMeta("SELECT participation_generation FROM issues WHERE id = ?")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"participation_generation"}).AddRow(1)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT 1 FROM wisps LIMIT 1")). + WillReturnRows(sqlmock.NewRows([]string{"1"})) + mock.ExpectQuery(`FROM dependencies WHERE issue_id IN`). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"issue_id", "depends_on_id", "type", "created_at", "created_by", "metadata", "thread_id"})) + mock.ExpectQuery(regexp.QuoteMeta("SELECT epoch FROM store_epoch WHERE id = 1")). + WillReturnRows(sqlmock.NewRows([]string{"epoch"}).AddRow(1)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(revision), 0) + 1 FROM issue_versions WHERE issue_id = ?")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"next"}).AddRow(1)) + mock.ExpectRollback() + + err := RecordVersionInTx(context.Background(), tx, id, "actor") + if !errors.Is(err, ErrIntegerNotRepresentable) { + t.Fatalf("RecordVersionInTx = %v, want ErrIntegerNotRepresentable", err) + } + if !strings.Contains(err.Error(), id) { + t.Errorf("refusal %q does not name the issue %s", err, id) + } + if !strings.Contains(err.Error(), tc.literal) { + t.Errorf("refusal %q does not name the offending literal %s", err, tc.literal) + } + if !strings.Contains(err.Error(), fmt.Sprintf("%q", tc.field)) { + t.Errorf("refusal %q does not name the field %q the number was found in", err, tc.field) + } + if rbErr := tx.Rollback(); rbErr != nil { + t.Fatalf("rolling back the writer's transaction: %v", rbErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the mint read or wrote something other than the scripted reads: %v", err) + } + }) + } +} + +// A legacy row (participation_generation NULL) holding the same out-of-range number is outside +// the mint's refusal: design §16.2b's write fence returns right after the snapshot, so the +// admission gate never runs and RecordVersionInTx returns nil. +// +// The mock scripts the snapshot, the labels and the fence's read returning NULL, and nothing after +// it. A mint that read the dependencies, the epoch or a revision, or wrote, before it decided to +// skip would fail the call with an unexpected-statement error instead of returning nil, so this is +// the cheapest pin of the fence's order against the gate. +func TestRecordVersionInTxSkipsALegacyRowBeforeTheAdmissionGate(t *testing.T) { + const id = "bd-9" + _, mock, tx := beginMockTx(t) + defer ScopeVersionedHistoryTransaction(tx, true)() + + values := issueRowValues(id, "a legacy row that carries an out-of-range number") + for i, col := range issueColumns() { + if col == "metadata" { + values[i] = `{"ts":1727000000000000000}` + } + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT " + IssueSelectColumns + " FROM issues " + sqlbuild.LeaseJoin("issues") + " WHERE id = ?")). + WithArgs(id). + WillReturnRows(issueRows().AddRow(values...)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT label FROM labels WHERE issue_id = ? ORDER BY label")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"label"})) + mock.ExpectQuery(regexp.QuoteMeta("SELECT participation_generation FROM issues WHERE id = ?")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"participation_generation"}).AddRow(nil)) + mock.ExpectRollback() + + if err := RecordVersionInTx(context.Background(), tx, id, "actor"); err != nil { + t.Fatalf("RecordVersionInTx on a legacy row = %v, want nil: the fence skips it before the admission gate", err) + } + if rbErr := tx.Rollback(); rbErr != nil { + t.Fatalf("rolling back the writer's transaction: %v", rbErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the mint read or wrote something after the fence skipped the row: %v", err) + } +} + +// TestRecordVersionAtInTxKeepsThePlaneRuleButNotTheFence pins what the +// test-support mint (RecordVersionAtInTx, mintUnfenced) does on the paths #6943 +// gave the shared body. +// +// - A row the wisps table holds, flags clear: the plane rule is not the fence, +// so the unfenced mint honours it like the other two kinds. The issues-table +// read finds nothing and the mint returns nil having written nothing, where +// before it would have inserted a version row no issues row points at. +// - An issues-table row flagged NoHistory: never versioned, and the unfenced +// mint never declares participation, so it returns right after the snapshot, +// without reading or stamping participation_generation. +// - A legacy row (participation_generation NULL), flags clear: the fence does +// not apply, so it mints, and the closing UPDATE advances current_revision +// alone, with no stamp. +// +// The mock scripts each sequence exactly, so an extra read or any write fails +// the call with an unexpected-statement error. +func TestRecordVersionAtInTxKeepsThePlaneRuleButNotTheFence(t *testing.T) { + const id = "bd-7" + at := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + issuesSnapshot := regexp.QuoteMeta("SELECT " + IssueSelectColumns + " FROM issues " + sqlbuild.LeaseJoin("issues") + " WHERE id = ?") + participation := regexp.QuoteMeta("SELECT participation_generation FROM issues WHERE id = ?") + + t.Run("a row the wisps table holds mints nothing", func(t *testing.T) { + _, mock, tx := beginMockTx(t) + mock.ExpectQuery(issuesSnapshot).WithArgs(id).WillReturnRows(issueRows()) + mock.ExpectQuery(regexp.QuoteMeta("SELECT " + IssueSelectColumns + " FROM wisps " + sqlbuild.LeaseJoin("wisps") + " WHERE id = ?")). + WithArgs(id). + WillReturnRows(issueRows().AddRow(issueRowValues(id, "a squashed wisp root, flags clear")...)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT label FROM wisp_labels WHERE issue_id = ? ORDER BY label")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"label"})) + mock.ExpectQuery(participation).WithArgs(id).WillReturnRows(sqlmock.NewRows([]string{"participation_generation"})) + mock.ExpectRollback() + + if err := RecordVersionAtInTx(context.Background(), tx, id, "actor", at); err != nil { + t.Fatalf("RecordVersionAtInTx on a wisps-table row = %v, want nil", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the unfenced mint did something after finding no issues row: %v", err) + } + }) + + t.Run("an issues row flagged no-history is neither minted nor stamped", func(t *testing.T) { + _, mock, tx := beginMockTx(t) + values := issueRowValues(id, "no-history, on the issues plane") + for i, col := range issueColumns() { + if col == "no_history" { + values[i] = int64(1) + } + } + mock.ExpectQuery(issuesSnapshot).WithArgs(id).WillReturnRows(issueRows().AddRow(values...)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT label FROM labels WHERE issue_id = ? ORDER BY label")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"label"})) + mock.ExpectRollback() + + if err := RecordVersionAtInTx(context.Background(), tx, id, "actor", at); err != nil { + t.Fatalf("RecordVersionAtInTx on a no-history row = %v, want nil", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the unfenced mint read or wrote something for a no-history row: %v", err) + } + }) + + t.Run("a legacy row is minted and left unstamped", func(t *testing.T) { + _, mock, tx := beginMockTx(t) + mock.ExpectQuery(issuesSnapshot).WithArgs(id). + WillReturnRows(issueRows().AddRow(issueRowValues(id, "legacy, history was off when it was made")...)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT label FROM labels WHERE issue_id = ? ORDER BY label")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"label"})) + mock.ExpectQuery(participation).WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"participation_generation"}).AddRow(nil)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT 1 FROM wisps LIMIT 1")). + WillReturnRows(sqlmock.NewRows([]string{"1"})) + mock.ExpectQuery(`FROM dependencies WHERE issue_id IN`). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"issue_id", "depends_on_id", "type", "created_at", "created_by", "metadata", "thread_id"})) + mock.ExpectQuery(regexp.QuoteMeta("SELECT epoch FROM store_epoch WHERE id = 1")). + WillReturnRows(sqlmock.NewRows([]string{"epoch"}).AddRow(1)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(revision), 0) + 1 FROM issue_versions WHERE issue_id = ?")). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"next"}).AddRow(1)) + mock.ExpectExec(`INSERT INTO issue_versions`). + WithArgs(id, int64(1), 1, sqlmock.AnyArg(), "actor", at, sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(regexp.QuoteMeta("UPDATE issues SET current_revision = ?, updated_at = updated_at WHERE id = ?")). + WithArgs(int64(1), id). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectRollback() + + if err := RecordVersionAtInTx(context.Background(), tx, id, "actor", at); err != nil { + t.Fatalf("RecordVersionAtInTx on a legacy row = %v, want nil", err) + } + if err := tx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the unfenced mint did not mint a legacy row exactly once, unstamped: %v", err) + } + }) +} + +// refusalClass names what an admission verdict is, so two functions can be +// compared for agreement without depending on their exact wording. +func refusalClass(err error) string { + switch { + case err == nil: + return "admitted" + case errors.Is(err, ErrIntegerNotRepresentable): + return "number outside the I-JSON range" + case strings.Contains(err.Error(), "Duplicate key"): + return "duplicate key" + default: + return "other refusal" + } +} + +func issueWithTimeout(timeout time.Duration) types.Issue { + return types.Issue{ID: "be-1", Title: "t", IssueType: types.TypeGate, Timeout: timeout} +} + +// The check the pre-enable scan runs and the check the mint runs must never +// disagree, or a store could pass the scan, be switched on, and then refuse the +// first write to the row the scan cleared. So the check is the mint's own two +// steps behind one exported name, and this pins that they agree on every shape +// of issue. +// +// The domain is the WHOLE issue, because that is what the mint canonicalizes and +// what the scan hands the check. Metadata covers a number past the range, a +// fraction rounding past it, duplicate keys at the top level and nested, +// ordinary values, null, empty and an empty object, and text that is not JSON at +// all. The other field that can reach the range is a gate's timeout, a +// time.Duration that encoding/json writes as nanoseconds: 2^53-1 ns is about +// 104.25 days, so 2500h is admitted and 2600h is not. +func TestCheckIssueVersionableAgreesWithTheMint(t *testing.T) { + const maxExact = 1<<53 - 1 // 9007199254740991 ns, the last whole nanosecond count in range + for _, tc := range []struct { + name string + issue types.Issue + want string + }{ + {"number past the range", issueWithMetadata(`{"ts":1727000000000000000}`), "number outside the I-JSON range"}, + {"fraction rounding past the range", issueWithMetadata(`{"n":9007199254740993.5}`), "number outside the I-JSON range"}, + {"overflowing exponent", issueWithMetadata(`{"n":1e400}`), "number outside the I-JSON range"}, + {"number nested in an array", issueWithMetadata(`{"a":[1,{"b":-9007199254740993}]}`), "number outside the I-JSON range"}, + {"duplicate key", issueWithMetadata(`{"k":1,"k":2}`), "duplicate key"}, + {"nested duplicate key", issueWithMetadata(`{"a":{"k":1,"k":2}}`), "duplicate key"}, + {"ordinary values", issueWithMetadata(`{"a":1,"b":[0.1,2,"x"],"c":{"d":null}}`), "admitted"}, + {"number at the bound", issueWithMetadata(`{"n":9007199254740991}`), "admitted"}, + {"tiny magnitude", issueWithMetadata(`{"n":1e-400}`), "admitted"}, + {"null", issueWithMetadata(`null`), "admitted"}, + {"empty object", issueWithMetadata(`{}`), "admitted"}, + {"empty", issueWithMetadata(``), "admitted"}, + {"text that is not JSON", issueWithMetadata(`{"a":`), "other refusal"}, + + {"no timeout", issueWithTimeout(0), "admitted"}, + {"timeout 2500h", issueWithTimeout(2500 * time.Hour), "admitted"}, + {"timeout 2600h", issueWithTimeout(2600 * time.Hour), "number outside the I-JSON range"}, + {"timeout 8760h", issueWithTimeout(8760 * time.Hour), "number outside the I-JSON range"}, + {"negative timeout -2600h", issueWithTimeout(-2600 * time.Hour), "number outside the I-JSON range"}, + {"timeout of exactly 2^53-1 ns", issueWithTimeout(time.Duration(maxExact)), "admitted"}, + {"timeout of exactly 2^53 ns", issueWithTimeout(time.Duration(maxExact + 1)), "number outside the I-JSON range"}, + {"timeout in range beside ordinary metadata", func() types.Issue { + issue := issueWithTimeout(time.Hour) + issue.Metadata = json.RawMessage(`{"ok":1}`) + return issue + }(), "admitted"}, + } { + t.Run(tc.name, func(t *testing.T) { + check := CheckIssueVersionable(&tc.issue) + _, mint := canonicalDurableState(tc.issue) + if got := refusalClass(check); got != tc.want { + t.Errorf("CheckIssueVersionable = %v (%s), want %s", check, got, tc.want) + } + if got := refusalClass(mint); got != tc.want { + t.Errorf("canonicalDurableState = %v (%s), want %s", mint, got, tc.want) + } + if refusalClass(check) != refusalClass(mint) { + t.Errorf("the pre-enable check and the mint disagree: check %v, mint %v", check, mint) + } + }) + } +} + +// The scan runs the check over every issue the mint would version and skips +// the ones it would not, by the mint's own rule (IsWisp): ephemeral rows and +// no-history rows are never versioned, so a poisoned value in one can never +// abort a write and is not an obstacle to switching history on. The scan +// reports what it found in the order it was given, names each offender with +// the refusal the mint would return and the field it was found in, and finds +// nothing in a clean store. +// +// A gate whose timeout is over 105 days is the case that motivated checking the +// whole issue: its timeout is past 2^53-1 ns, so, if it participates in history, +// the mint refuses every write to it, and a scan that looked only at metadata +// cleared the switch over it. +func TestFindUnversionableFindsWhatTheMintWouldRefuse(t *testing.T) { + oversize := json.RawMessage(`{"ts":1727000000000000000}`) + overlong := 105 * 24 * time.Hour + issues := []*types.Issue{ + {ID: "bd-1", Metadata: oversize}, + {ID: "bd-2", Metadata: json.RawMessage(`{"k":1,"k":2}`)}, + {ID: "bd-3", Metadata: json.RawMessage(`{"ok":1}`)}, + {ID: "bd-4", Ephemeral: true, Metadata: oversize}, + {ID: "bd-5", NoHistory: true, Metadata: oversize}, + {ID: "bd-6"}, + {ID: "bd-7", Metadata: json.RawMessage(`{"n":9007199254740993.5}`)}, + {ID: "bd-8", Ephemeral: true, WispPlaneOverride: boolPtr(false), Metadata: oversize}, + {ID: "bd-9", WispPlaneOverride: boolPtr(true), Metadata: oversize}, + {ID: "bd-10", IssueType: types.TypeGate, Timeout: overlong}, + {ID: "bd-11", IssueType: types.TypeGate, Timeout: 2500 * time.Hour}, + {ID: "bd-12", IssueType: types.TypeGate, Ephemeral: true, Timeout: overlong}, + {ID: "bd-13", IssueType: types.TypeGate, Timeout: -overlong}, + } + + got := FindUnversionable(issues) + + want := []struct { + id string + class string + field string + }{ + {"bd-1", "number outside the I-JSON range", "metadata"}, + {"bd-2", "duplicate key", "metadata"}, + {"bd-7", "number outside the I-JSON range", "metadata"}, + // The override decides which plane the mint routes an issue to, and the + // scan follows it: bd-8 is versioned despite Ephemeral, bd-9 is not. + {"bd-8", "number outside the I-JSON range", "metadata"}, + {"bd-10", "number outside the I-JSON range", "timeout"}, + {"bd-13", "number outside the I-JSON range", "timeout"}, + } + if len(got) != len(want) { + t.Fatalf("FindUnversionable found %d issues %v, want %d", len(got), unversionableIDs(got), len(want)) + } + for i, w := range want { + if got[i].ID != w.id { + t.Fatalf("offender %d is %s, want %s (order of the input); got %v", i, got[i].ID, w.id, unversionableIDs(got)) + } + if class := refusalClass(got[i].Err); class != w.class { + t.Errorf("%s: refusal class %q (%v), want %q", w.id, class, got[i].Err, w.class) + } + if got[i].Field != w.field { + t.Errorf("%s: field %q, want %q", w.id, got[i].Field, w.field) + } + if !strings.Contains(got[i].Err.Error(), fmt.Sprintf("%q", w.field)) { + t.Errorf("%s: refusal %q does not name the field %q", w.id, got[i].Err, w.field) + } + } + + clean := []*types.Issue{ + {ID: "bd-1", Metadata: json.RawMessage(`{"ok":1,"f":0.1}`)}, + {ID: "bd-2"}, + {ID: "bd-3", Metadata: json.RawMessage(`null`)}, + {ID: "bd-4", Ephemeral: true, Metadata: oversize}, + {ID: "bd-5", IssueType: types.TypeGate, Timeout: 2500 * time.Hour}, + {ID: "bd-6", IssueType: types.TypeGate, Ephemeral: true, Timeout: overlong}, + } + if found := FindUnversionable(clean); len(found) != 0 { + t.Fatalf("a store with nothing the mint would refuse reported %v", unversionableIDs(found)) + } + if found := FindUnversionable(nil); len(found) != 0 { + t.Fatalf("no issues reported %v", unversionableIDs(found)) + } +} + +func unversionableIDs(found []UnversionableIssue) []string { + ids := make([]string, 0, len(found)) + for _, f := range found { + ids = append(ids, f.ID) + } + return ids +} + +// The scan does not hand the check the issue the mint sees. It reads a lite +// projection (the heavy text columns are left out, labels are not loaded) and +// never loads dependencies, which the mint adds before it canonicalizes. That is +// sound only while everything the scan leaves out is free of numbers: a number in +// a field the scan never loads is one it can never refuse and the mint will. The +// next three tests are how that stays true. They fail, with the way out, the day +// a field is added that breaks it. + +// fieldByJSONName finds the exported field of the struct type typ that +// encoding/json writes under name. +func fieldByJSONName(typ reflect.Type, name string) (reflect.StructField, bool) { + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + if !f.IsExported() { + continue + } + tag := f.Tag.Get("json") + if tag == "-" { + continue + } + jsonName := strings.Split(tag, ",")[0] + if jsonName == "" { + jsonName = f.Name + } + if jsonName == name { + return f, true + } + } + return reflect.StructField{}, false +} + +// heavyColumnProblems maps each column to its field of the issue type and reports +// every one that is missing or that can hold a number. A string or a slice of +// strings cannot: encoding/json writes both without a JSON number in them. +func heavyColumnProblems(columns []string, issue reflect.Type) []string { + var problems []string + for _, col := range columns { + f, ok := fieldByJSONName(issue, col) + if !ok { + problems = append(problems, fmt.Sprintf("column %q has no %s field written under that name: map it, then say whether it can hold a number", col, issue)) + continue + } + numberFree := f.Type.Kind() == reflect.String || + (f.Type.Kind() == reflect.Slice && f.Type.Elem().Kind() == reflect.String) + if !numberFree { + problems = append(problems, fmt.Sprintf("column %q is %s.%s (%s), which can hold a number the scan never reads but the mint refuses: load it in the scan or classify it lite", col, issue, f.Name, f.Type)) + } + } + return problems +} + +// TestTheScanProjectionOmitsOnlyNumberFreeFields pins that every column the +// scan's lite projection leaves out (HeavyDropList) is a string or a list of +// strings on the issue. +func TestTheScanProjectionOmitsOnlyNumberFreeFields(t *testing.T) { + if problems := heavyColumnProblems(HeavyDropList, reflect.TypeOf(types.Issue{})); len(problems) > 0 { + t.Fatalf("the scan's projection leaves out a column that can hold a number:\n %s", strings.Join(problems, "\n ")) + } +} + +// The guard above would pass vacuously if it never found anything to refuse, so +// it is run once against columns that must be refused: numbers, metadata (a +// verbatim JSON document), a column that maps to nothing, and the columns that +// really are number-free. +func TestTheHeavyColumnGuardRefusesWhatCanHoldANumber(t *testing.T) { + issue := reflect.TypeOf(types.Issue{}) + for _, col := range []string{"priority", "estimated_minutes", "timeout", "metadata", "created_at_unix"} { + if problems := heavyColumnProblems([]string{col}, issue); len(problems) != 1 { + t.Errorf("column %q: got %d problems %v, want exactly one", col, len(problems), problems) + } + } + for _, col := range []string{"description", "design", "acceptance_criteria", "notes", "waiters", "payload"} { + if problems := heavyColumnProblems([]string{col}, issue); len(problems) != 0 { + t.Errorf("column %q is number-free but was refused: %v", col, problems) + } + } + problems := heavyColumnProblems([]string{"priority"}, issue) + if len(problems) == 1 && !strings.Contains(problems[0], "load it in the scan or classify it lite") { + t.Errorf("the refusal %q does not say how to resolve it", problems[0]) + } +} + +var ( + timeType = reflect.TypeOf(time.Time{}) + rawMessageType = reflect.TypeOf(json.RawMessage(nil)) +) + +// numberBearingPaths walks typ the way encoding/json does and returns the path of +// everything that can write a JSON number or a value this walk cannot see into: +// an integer, unsigned integer or float (a time.Duration is an int64), a +// json.RawMessage, an interface, a map. It recurses through slices, arrays, +// pointers and structs, skips what json:"-" leaves out and the unexported fields, +// and treats a time.Time as the string it is written as. +func numberBearingPaths(typ reflect.Type, path string, onPath map[reflect.Type]bool) []string { + switch typ { + case timeType: + return nil + case rawMessageType: + return []string{path + " is a json.RawMessage, passed through verbatim"} + } + switch typ.Kind() { + case reflect.Bool, reflect.String: + return nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64: + return []string{fmt.Sprintf("%s is a %s", path, typ)} + case reflect.Interface, reflect.Map: + return []string{fmt.Sprintf("%s is a %s", path, typ.Kind())} + case reflect.Pointer, reflect.Slice, reflect.Array: + return numberBearingPaths(typ.Elem(), path, onPath) + case reflect.Struct: + if onPath[typ] { + return nil + } + onPath[typ] = true + defer delete(onPath, typ) + var found []string + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + if !f.IsExported() || f.Tag.Get("json") == "-" { + continue + } + found = append(found, numberBearingPaths(f.Type, path+"."+f.Name, onPath)...) + } + return found + default: + return []string{fmt.Sprintf("%s is a %s, which this walk cannot classify", path, typ.Kind())} + } +} + +// TestWhatTheMintAddsAndTheScanDoesNotLoadCarriesNoNumber covers the other half +// of what the scan leaves out. The mint adds the issue's dependencies before it +// canonicalizes, and the scan reads without labels, so a number anywhere in a +// Dependency, a Comment, a BondRef or a label would be refused by the mint and +// invisible to the scan. None carries one today; a field that could (a Weight +// int on Dependency) fails here, and the person who adds it decides whether the +// scan has to load it. +func TestWhatTheMintAddsAndTheScanDoesNotLoadCarriesNoNumber(t *testing.T) { + for _, tc := range []struct { + name string + typ reflect.Type + }{ + {"types.Dependency", reflect.TypeOf(types.Dependency{})}, + {"types.Comment", reflect.TypeOf(types.Comment{})}, + {"types.BondRef", reflect.TypeOf(types.BondRef{})}, + {"a label", reflect.TypeOf("")}, + {"the labels of an issue", reflect.TypeOf([]string(nil))}, + } { + t.Run(tc.name, func(t *testing.T) { + if found := numberBearingPaths(tc.typ, tc.name, map[reflect.Type]bool{}); len(found) > 0 { + t.Fatalf("a field the mint writes and the scan does not load can hold a number:\n %s", strings.Join(found, "\n ")) + } + }) + } +} + +// The walk is run against shapes it must flag and shapes it must not, for the +// same reason as the guard above. +func TestTheNumberFreeWalkFlagsEveryNumberBearingShape(t *testing.T) { + type weighted struct { + Weight int `json:"weight"` + } + type unsigned struct{ N uint32 } + type scored struct{ Score float64 } + type waiting struct{ Wait time.Duration } + type verbatim struct{ Blob json.RawMessage } + type anything struct{ Value any } + type keyed struct{ Extra map[string]string } + type nested struct{ Deps []*weighted } + type tagged struct { + Name string `json:"name"` + Tags []string `json:"tags"` + At time.Time `json:"at"` + Skipped int `json:"-"` + _ int + } + type recursive struct{ Next *recursive } + + for name, typ := range map[string]reflect.Type{ + "an int": reflect.TypeOf(weighted{}), + "an unsigned integer": reflect.TypeOf(unsigned{}), + "a float": reflect.TypeOf(scored{}), + "a time.Duration": reflect.TypeOf(waiting{}), + "a json.RawMessage": reflect.TypeOf(verbatim{}), + "an interface": reflect.TypeOf(anything{}), + "a map": reflect.TypeOf(keyed{}), + "an int behind slice+pointer": reflect.TypeOf(nested{}), + } { + if found := numberBearingPaths(typ, "x", map[reflect.Type]bool{}); len(found) == 0 { + t.Errorf("%s was not flagged", name) + } + } + for name, typ := range map[string]reflect.Type{ + "strings, a time, a skipped int and an unexported int": reflect.TypeOf(tagged{}), + "a type that contains itself": reflect.TypeOf(recursive{}), + } { + if found := numberBearingPaths(typ, "x", map[reflect.Type]bool{}); len(found) != 0 { + t.Errorf("%s was flagged: %v", name, found) + } + } +} + +// There is ONE check, CheckIssueVersionable, and the scan and the mint both go +// through it. A check over a part of the issue, the metadata alone as there once +// was, can pass what the mint refuses, and an exported one invites reuse by the +// next caller (a doctor check, say) that then inherits the gap. So no exported +// function in the package may be named for versionability beside the check and +// the scan that calls it. +func TestNoExportedCheckOverASubPartOfTheIssueExistsBesideCheckIssueVersionable(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("parse issueops package: %v", err) + } + allowed := map[string]bool{"CheckIssueVersionable": true, "FindUnversionable": true} + seen := map[string]bool{} + var stray []string + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv != nil || !fn.Name.IsExported() { + continue + } + if !strings.Contains(strings.ToLower(fn.Name.Name), "versionable") { + continue + } + seen[fn.Name.Name] = true + if !allowed[fn.Name.Name] { + stray = append(stray, fmt.Sprintf("%s (%s)", fn.Name.Name, fset.Position(fn.Pos()))) + } + } + } + } + sort.Strings(stray) + if len(stray) > 0 { + t.Errorf("an exported check over part of the issue exists beside CheckIssueVersionable: %s", strings.Join(stray, ", ")) + } + for name := range allowed { + if !seen[name] { + t.Errorf("%s was not found: this guard no longer sees the check it protects", name) + } + } +} diff --git a/internal/storage/issueops/version_history_scan.go b/internal/storage/issueops/version_history_scan.go new file mode 100644 index 000000000..855869325 --- /dev/null +++ b/internal/storage/issueops/version_history_scan.go @@ -0,0 +1,53 @@ +package issueops + +import ( + "errors" + + "github.com/steveyegge/beads/internal/types" +) + +// UnversionableIssue names one issue that RecordVersionInTx would refuse to +// version, and the refusal it would return. Field is the top-level field of the +// issue the refusal was found in ("metadata", "timeout"), empty when the refusal +// does not name one. +type UnversionableIssue struct { + ID string + Field string + Err error +} + +// FindUnversionable runs CheckIssueVersionable over every issue in issues that +// the mint would version and returns the ones it refuses, in the order they were +// given. It is the core of the check made when versioned history is switched on: +// with history on, a write that introduces a value the mint refuses aborts in its +// own transaction, but a row that already holds one -- written while history was +// off, or by a path that does not mint -- would, if it participates in history, +// fail every later write to it. The check is over the whole issue, as the mint's +// is, and not over its metadata alone: a gate's timeout past 2^53-1 nanoseconds is +// refused too. +// +// It skips the rows the mint never versions by IsWisp: ephemeral and no-history +// rows are never versioned, so a value they hold can never abort a write. It +// cannot skip legacy rows, because participation is not on types.Issue, so on a +// store with the write fence (design §16.2b) it is a deliberate +// over-approximation: the mint skips a legacy row whatever it holds, and this +// still refuses over it, which is the safe direction because it can only refuse +// more. It does no I/O; the caller loads the rows, which is what lets one +// function serve the direct store and the proxied route. +func FindUnversionable(issues []*types.Issue) []UnversionableIssue { + var found []UnversionableIssue + for _, issue := range issues { + if issue == nil || IsWisp(issue) { + continue + } + if err := CheckIssueVersionable(issue); err != nil { + var named *refusedFieldError + field := "" + if errors.As(err, &named) { + field = named.field + } + found = append(found, UnversionableIssue{ID: issue.ID, Field: field, Err: err}) + } + } + return found +} diff --git a/internal/storage/issueops/version_list.go b/internal/storage/issueops/version_list.go new file mode 100644 index 000000000..fdfc9f2f9 --- /dev/null +++ b/internal/storage/issueops/version_list.go @@ -0,0 +1,77 @@ +package issueops + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage" +) + +// ListVersionsInTx returns every retained version of issueID, newest first. +// +// No subquery wrapper here, deliberately: HistoryInTx wraps its select +// because dolt_history_issues is a Dolt SYSTEM table and the planner's +// max1Row optimization mis-assumes WHERE id=? yields one row. issue_versions +// is an ordinary user table with a composite (issue_id, revision) primary +// key, so that optimization does not apply and the wrapper would only cost a +// materialization. +// +// An issue with no versions returns an empty slice and no error -- that is a +// truthful "nothing retained here", and it is the CALLER's job to distinguish +// it from "the feature is off" and from "no such issue", which it cannot do +// from this result alone. +// +// A removed version's RemovedRestriction is one of the as-of read's four values: +// whatever else the column holds, a NULL or empty one included, is reported as +// unknown, exactly as AsOfReadInTx reports it. A version that was not removed +// carries none. +func ListVersionsInTx(ctx context.Context, tx DBTX, issueID string) ([]storage.IssueVersion, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT + issue_id, revision, epoch, + COALESCE(change_actor, '') AS change_actor, + COALESCE(change_agent, '') AS change_agent, + COALESCE(change_message, '') AS change_message, + change_at, + COALESCE(attribution_status, '') AS attribution_status, + removed_at, + COALESCE(removed_reason, '') AS removed_reason, + COALESCE(removed_restriction, '') AS removed_restriction, + COALESCE(LENGTH(durable_state), 0) AS state_bytes + FROM issue_versions + WHERE issue_id = ? + ORDER BY revision DESC + `, issueID) + if err != nil { + return nil, fmt.Errorf("failed to list issue versions: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.IssueVersion + for rows.Next() { + var v storage.IssueVersion + var removedAt sql.NullTime + if err := rows.Scan( + &v.IssueID, &v.Revision, &v.Epoch, + &v.ChangeActor, &v.ChangeAgent, &v.ChangeMessage, &v.ChangeAt, + &v.AttributionStatus, + &removedAt, &v.RemovedReason, &v.RemovedRestriction, + &v.StateBytes, + ); err != nil { + return nil, fmt.Errorf("failed to scan issue version: %w", err) + } + if removedAt.Valid { + t := removedAt.Time + v.RemovedAt = &t + v.RemovedRestriction = string(NormalizeRemovedRestriction(v.RemovedRestriction)) + } else { + v.RemovedRestriction = "" + } + out = append(out, v) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to read issue versions: %w", err) + } + return out, nil +} diff --git a/internal/storage/leg_contract_wiring_test.go b/internal/storage/leg_contract_wiring_test.go index 78ac5200b..00fdcdd46 100644 --- a/internal/storage/leg_contract_wiring_test.go +++ b/internal/storage/leg_contract_wiring_test.go @@ -33,6 +33,30 @@ var neverSatisfiedTags = map[string]bool{"ignore": true, "never": true} const importerOneAccessorWaiverReason = "Importer has one accessor (uow.ImporterSource); " + "the store legs run bd import through the raw seam and implement no Importer role" +// asOfReadNoHTTPWireWaiverReason covers the five R7.1 as-of read contracts +// (backend/conformance/versioned_read_contract.go). Unlike the +// participation-generation cases, which run on this leg because their subject +// is a create or an update the client can send +// (internal/httpclient/served_participation_generation_test.go), the subject +// here is the read itself, AsOfReadFixture.Resolve, and it has no form on the +// wire: +// - R7.1's read is issueops.AsOfReadInTx. No publicops role reaches it and no +// wire operation serves it, so there is nothing for the client to call. +// - The wire carries neither selector (an instant, or a previously served +// version address) nor the gone/unknown refusal the read answers with. +// - The client's only point-in-time read, storage.DoltStorage.AsOf, is a +// Dolt commit-ref read, not R7.1, and it is on the client's unsupported +// allowlist (D8: v0 publishes no history surface). +// +// Binding Resolve to the reference store behind the served env would run the +// cases green, but as a property of that store, never of the client; the store +// legs already pin it. The waiver stands until the wire publishes an as-of +// read. +const asOfReadNoHTTPWireWaiverReason = "the read under test has no form on the http wire: R7.1's as-of read " + + "(issueops.AsOfReadInTx) is reached by no publicops role and no wire operation, the wire carries neither an " + + "instant/version-address selector nor a gone/unknown refusal, and the client's only point-in-time read, " + + "DoltStorage.AsOf, is a Dolt commit-ref read on its unsupported allowlist (D8)" + // The waivers of the three legs registered here. Each is registered beside its // leg rather than written into one literal, so a leg registered elsewhere // brings its waivers the same way; see unwiredContractEntrypoints. @@ -57,6 +81,12 @@ func init() { "RunIssueOperationsCreateParentChildRecomputesWaitsForClosure": stagingWaiverReason, }) registerContractLegWaivers("http", map[string]string{ + "RunAsOfReadIsCompleteOnlyForTheAnsweringStoresHoldings": asOfReadNoHTTPWireWaiverReason, + "RunAsOfReadRefusesAsGoneOrUnknownNeverSubstitutingAnotherVersion": asOfReadNoHTTPWireWaiverReason, + "RunAsOfReadReportsTheAddressItServed": asOfReadNoHTTPWireWaiverReason, + "RunAsOfReadReturnsTheLatestVersionAtOrBeforeTStrictlyExcludingLater": asOfReadNoHTTPWireWaiverReason, + "RunAsOfReadSelectorAcceptsEitherAnInstantOrAVersionAddress": asOfReadNoHTTPWireWaiverReason, + "RunCycleDetectorIncludeTracksFindsTheMoleculeRootShape": cycleTracksWireGapWaiverReason, "RunCycleDetectorIncludeTracksIgnoresAPureTracksLoop": cycleTracksWireGapWaiverReason, "RunCycleDetectorIncludeTracksWalksEachEdgeInItsStoredDirection": cycleTracksWireGapWaiverReason, diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 09be66e8f..f81aca9bc 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -120,6 +120,7 @@ go_library( "migrations/0069_widen_issue_versions_datetime_precision.up.sql", "migrations/ignored/0028_add_wisps_participation_generation.up.sql", "migrations/0070_add_participation_generation.up.sql", + "migrations/0071_add_removed_restriction.up.sql", ], importpath = "github.com/steveyegge/beads/internal/storage/schema", visibility = ["//:__subpackages__"], @@ -154,6 +155,7 @@ go_test( "migration_0068_add_attribution_status_test.go", "migration_0069_datetime_precision_test.go", "migration_0070_add_participation_generation_test.go", + "migration_0071_add_removed_restriction_test.go", "migration_body_drain_test.go", "migration_freeze_test.go", "migration_watchdog_test.go", @@ -169,6 +171,7 @@ go_test( deps = [ "//internal/debug", "//internal/storage/depid", + "//internal/storage/issueops", "//internal/storage/rowid", "//internal/testutil", "@com_github_data_dog_go_sqlmock//:go-sqlmock", diff --git a/internal/storage/schema/cli_migrations.go b/internal/storage/schema/cli_migrations.go index 8b595c17a..1d161cbc6 100644 --- a/internal/storage/schema/cli_migrations.go +++ b/internal/storage/schema/cli_migrations.go @@ -168,6 +168,15 @@ func cliCompatibleMigrationSQL(name, sqlText string) string { // substitute ALTERs wisps directly, the same as 0067's, so it // belongs on cliSubstituteAssumesWispTables too. return cliMigration0070AddParticipationGeneration + case "0071_add_removed_restriction.up.sql": + // Direct DDL for the same reason as 0067/0068: the source migration's + // PREPARE guard (an INFORMATION_SCHEMA probe) is what makes the raw + // .up.sql idempotent when replayed onto an already-migrated store, + // and the 2.2.x CLI no-ops a prepared ADD COLUMN. issue_versions is + // always present here -- 0067 runs earlier in the same fresh-bundle + // series -- and never carries removed_restriction yet, so the column + // always needs adding on a fresh bundle. + return cliMigration0071AddRemovedRestriction default: return sqlText } @@ -318,6 +327,12 @@ ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME(6);` const cliMigration0070AddParticipationGeneration = `ALTER TABLE issues ADD COLUMN participation_generation BIGINT NULL; ALTER TABLE wisps ADD COLUMN participation_generation BIGINT NULL;` +// cliMigration0071AddRemovedRestriction is 0071 with its one guarded PREPARE +// block replaced by the direct ALTER it would run on a fresh database. +// issue_versions is created earlier in the same series by 0067 and never +// carries removed_restriction yet, so the column always needs adding here. +const cliMigration0071AddRemovedRestriction = `ALTER TABLE issue_versions ADD COLUMN removed_restriction VARCHAR(30);` + const cliMigration0041SplitDependenciesTarget = `DELETE FROM dolt_nonlocal_tables; CALL DOLT_COMMIT('-Am', 'disable nonlocal tables for fk migrations'); SET FOREIGN_KEY_CHECKS = 0; diff --git a/internal/storage/schema/migration_0069_datetime_precision_test.go b/internal/storage/schema/migration_0069_datetime_precision_test.go index ea6bcc8cd..503be1b6d 100644 --- a/internal/storage/schema/migration_0069_datetime_precision_test.go +++ b/internal/storage/schema/migration_0069_datetime_precision_test.go @@ -39,6 +39,12 @@ const migration0069Down = "0069_widen_issue_versions_datetime_precision.down.sql const migration0069ChangeAtDatetimeGuard = "@issue_versions_change_at_needs_widen" const migration0069RemovedAtDatetimeGuard = "@issue_versions_removed_at_needs_widen" +// TestLatestVersionIncludesMigration0069 (pinning LatestVersion() == 69) is +// superseded by TestLatestVersionIncludesMigration0070 +// (migration_0070_add_participation_generation_test.go) now that 0070 claims the +// next free slot; only one such pin lives at a time, matching how this +// test itself already superseded 0068's own version. + // TestMigration0069WidensChangeAtAndRemovedAtPrecision is the pure-Go, // DB-independent half of the pin, mirroring // TestMigration0068AddsAttributionStatus's shape: it checks the frozen diff --git a/internal/storage/schema/migration_0070_add_participation_generation_test.go b/internal/storage/schema/migration_0070_add_participation_generation_test.go index e6d5a7369..0993347f2 100644 --- a/internal/storage/schema/migration_0070_add_participation_generation_test.go +++ b/internal/storage/schema/migration_0070_add_participation_generation_test.go @@ -33,20 +33,11 @@ import ( const migration0070Up = "0070_add_participation_generation.up.sql" const migration0070Down = "0070_add_participation_generation.down.sql" -// TestLatestVersionIncludesMigration0070 pins the real next free slot this -// migration claims, superseding 0069's own version of this test -// (LatestVersion() moved from 69 to 70 the moment this migration file was -// added). Deliberately a hardcoded literal for the same reason 0067's, -// 0068's, and 0069's were: LatestVersion() drifting to 70 for the wrong -// reason (an unrelated migration landing first) should still be caught by -// this test failing to explain why 70 is participation_generation-shaped, -// which the CLI test below checks. -func TestLatestVersionIncludesMigration0070(t *testing.T) { - const want = 70 - if got := LatestVersion(); got != want { - t.Fatalf("LatestVersion() = %d, want %d (issues/wisps participation_generation migration slot claimed by be-h89oq)", got, want) - } -} +// TestLatestVersionIncludesMigration0070 (pinning LatestVersion() == 70) is +// superseded by TestLatestVersionIncludesMigration0071 +// (migration_0071_add_removed_restriction_test.go) now that 0071 claims the +// next free slot; only one such pin lives at a time, matching how this +// test itself already superseded 0069's own version. // TestMigration0070AddsParticipationGeneration is the pure-Go, DB-independent // half of the pin, mirroring TestMigration0069WidensChangeAtAndRemovedAtPrecision's diff --git a/internal/storage/schema/migration_0071_add_removed_restriction_test.go b/internal/storage/schema/migration_0071_add_removed_restriction_test.go new file mode 100644 index 000000000..2b4da39ac --- /dev/null +++ b/internal/storage/schema/migration_0071_add_removed_restriction_test.go @@ -0,0 +1,171 @@ +package schema + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/testutil" +) + +// R7.1 as-of read (gastownhall/beads#5898 revision 9, gastownhall/beads#6136) +// adds one nullable column: issue_versions gains +// removed_restriction VARCHAR(30). issue_versions already carries removed_at +// and removed_reason (migration 0067), virgin and unused until this slice; +// AsOfReadInTx (internal/storage/issueops/asof_read.go) reads all three as +// the durable "this version row was removed" marker. removed_restriction +// carries the categorical restriction removed_at's presence alone cannot: +// gone_retention / gone_erasure / gone_reorganization / unknown, never +// live -- live is the absence of a value, not a stored one. + +const migration0071Up = "0071_add_removed_restriction.up.sql" +const migration0071Down = "0071_add_removed_restriction.down.sql" + +// TestLatestVersionIncludesMigration0071 pins the real next free slot this +// phase claims, superseding 0070's own version of this test +// (migration_0070_add_participation_generation_test.go). Deliberately a +// hardcoded literal for the same reason 0067's, 0068's, 0069's and 0070's +// were: LatestVersion() drifting to 71 for the wrong reason should still be +// caught by this test failing to explain why 71 is removed-restriction-shaped, +// which the CLI test below checks. +func TestLatestVersionIncludesMigration0071(t *testing.T) { + const want = 71 + if got := LatestVersion(); got != want { + t.Fatalf("LatestVersion() = %d, want %d (the migration slot claimed for issue_versions.removed_restriction)", got, want) + } +} + +// TestMigration0071AddsRemovedRestriction is a pure-Go, DB-independent check +// of the frozen migration bytes themselves — it runs even where no `dolt` +// binary is available. +// +// It pins the same guarded-PREPARE shape 0067/0068 use and for the same +// reason: the guard is what makes a raw replay of this file onto an +// already-migrated store a no-op, and Dolt accepts no unprepared conditional +// ADD COLUMN. The cost is the same pre-2.3 CLI hazard those migrations carry, +// so this migration needs the same direct-DDL override in +// cliCompatibleMigrationSQL, policed by the same two guard tests +// (TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified and +// TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities). +func TestMigration0071AddsRemovedRestriction(t *testing.T) { + upSQL, err := MigrationSQL(migration0071Up) + if err != nil { + t.Fatalf("MigrationSQL(%s) error = %v, want the migration file to exist", migration0071Up, err) + } + for _, want := range []string{ + "ALTER TABLE issue_versions ADD COLUMN removed_restriction VARCHAR(30)", + "COLUMN_NAME = 'removed_restriction'", + "@issue_versions_rr_needs_add", + } { + if !strings.Contains(upSQL, want) { + t.Errorf("0071 up migration missing %q\nfull SQL:\n%s", want, upSQL) + } + } + if !strings.Contains(strings.ToUpper(upSQL), "PREPARE STMT FROM @SQL") { + t.Error("0071 up migration must keep its guarded PREPARE block — it is what makes a raw .up.sql replay onto an already-migrated store a no-op, and Dolt accepts no unprepared conditional ADD COLUMN. Unwrapping it also invalidates cliMigration0071AddRemovedRestriction.") + } + // The bundle override is what keeps the PREPARE above off the pre-2.3 + // CLI path. Assert it directly rather than trusting the two schema_test + // assertions to stay pointed at this migration. + const wantDirectDDL = "ALTER TABLE issue_versions ADD COLUMN removed_restriction VARCHAR(30);" + if !strings.Contains(cliCompatibleMigrationSQL(migration0071Up, upSQL), wantDirectDDL) { + t.Errorf("0071's CLI bundle substitute missing direct DDL %q", wantDirectDDL) + } + if cliSubstituteAssumesWispTables(migration0071Up) { + t.Error("0071's CLI substitute touches only issue_versions, which has no wisps-side counterpart table — it must not be listed in cliSubstituteAssumesWispTables") + } + + // down.sql files are not part of the embedded FS (only migrations/*.up.sql + // is //go:embed'd — see mainSource.files), so unlike the up side above, + // this reads straight from disk by package-relative path, matching + // TestMigration0067AddsVersionedBeadsSchema's / TestMigration0068AddsAttributionStatus's + // precedent. + downBytes, err := os.ReadFile("migrations/" + migration0071Down) + if err != nil { + t.Fatalf("read %s: %v, want the migration file to exist", migration0071Down, err) + } + downSQL := string(downBytes) + for _, want := range []string{ + "ALTER TABLE issue_versions DROP COLUMN removed_restriction", + "COLUMN_NAME = 'removed_restriction'", + } { + if !strings.Contains(downSQL, want) { + t.Errorf("0071 down migration missing %q\nfull SQL:\n%s", want, downSQL) + } + } + // Only migrations/*.up.sql is embedded into the CLI fresh bundle + // (mainSource.files), so the pre-2.3 prepared-DDL hazard never reaches a + // down migration and the guard is free — 0060's/0067's/0068's downs are + // the precedent. + if !strings.Contains(strings.ToUpper(downSQL), "PREPARE STMT FROM @SQL") { + t.Error("0071 down migration must guard its DROP COLUMN the way the up migration guards its ADD COLUMN, so a partially-applied or already-rolled-back workspace rolls back safely") + } +} + +// TestMigration0071HeaderSpellsTheVocabularyTheCodeStores pins the header's +// vocabulary to the constants that store, read and label the column (finding 6 +// of the #6661 review). The header is the one place a writer learns which +// strings to store, and a writer follows it literally: it said gone-retention, +// the code compares gone_retention, so a row written from the header was not +// recognised and rendered without a label. Deriving the expected spellings +// from the constants keeps the two from drifting apart again. +// +// Pure Go, like TestMigration0071AddsRemovedRestriction: it reads the frozen +// bytes and needs no `dolt` binary. +func TestMigration0071HeaderSpellsTheVocabularyTheCodeStores(t *testing.T) { + upSQL, err := MigrationSQL(migration0071Up) + if err != nil { + t.Fatalf("MigrationSQL(%s) error = %v, want the migration file to exist", migration0071Up, err) + } + for _, stored := range []issueops.AsOfRestriction{ + issueops.AsOfRestrictionGoneRetention, + issueops.AsOfRestrictionGoneErasure, + issueops.AsOfRestrictionGoneReorganization, + issueops.AsOfRestrictionUnknown, + } { + if !strings.Contains(upSQL, string(stored)) { + t.Errorf("0071 header never spells the stored value %q", stored) + } + if hyphenated := strings.ReplaceAll(string(stored), "_", "-"); hyphenated != string(stored) && strings.Contains(upSQL, hyphenated) { + t.Errorf("0071 header spells %q as %q; the code stores and compares the underscore spelling, so a writer following the header stores a value that renders unlabeled", stored, hyphenated) + } + } +} + +// TestMigration0071AddsRemovedRestrictionThroughDoltCLI applies the full +// migration bundle through a real `dolt` binary (skipped without one — see +// testutil.RequireDoltBinary) and checks the shape acceptance criteria a +// pure-Go SQL-text check cannot: actual column type/nullability as Dolt +// reports it, and that the column stays NULL for a row that never sets it +// (Live is the absence of a value) while round-tripping any of the four +// restriction strings a row does set it to. +func TestMigration0071AddsRemovedRestrictionThroughDoltCLI(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := t.TempDir() + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + runDoltSQL(t, dir, AllMigrationsSQL()) + + requireDoltColumnShape(t, dir, "issue_versions", "removed_restriction", "varchar(30)", "YES") + requireDoltNoRows(t, dir, "SELECT issue_id FROM issue_versions", "issue_versions") + + runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:00', 'claimed')`) + // issue_id rides along as a non-NULL anchor column: queryDoltCSV trims the + // whole `dolt sql -r csv` output before parsing, and a row whose ONLY + // selected column is NULL renders as a bare blank line, which + // strings.TrimSpace swallows when it's the trailing line -- silently + // turning "one row, NULL value" into "zero rows" and passing the wrong + // assertion for the wrong reason. Selecting issue_id alongside keeps the + // row's CSV line non-blank so it survives the trim. + rows := queryDoltCSV(t, dir, `SELECT issue_id, removed_restriction FROM issue_versions WHERE issue_id = 'iv-1'`) + if len(rows) != 1 || rows[0]["removed_restriction"] != "" { + t.Fatalf("removed_restriction for a row that never set it = %v, want NULL/empty (Live is the absence of a value)", rows) + } + + runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status, removed_at, removed_reason, removed_restriction) VALUES ('iv-2', 1, 1, '2026-09-01 00:00:00', 'claimed', '2026-09-02 00:00:00', 'retention window elapsed', 'gone_retention')`) + rows = queryDoltCSV(t, dir, `SELECT issue_id, removed_restriction FROM issue_versions WHERE issue_id = 'iv-2'`) + if len(rows) != 1 || rows[0]["removed_restriction"] != "gone_retention" { + t.Fatalf("removed_restriction round-trip failed post-migration: %v", rows) + } +} diff --git a/internal/storage/schema/migrations/0071_add_removed_restriction.down.sql b/internal/storage/schema/migrations/0071_add_removed_restriction.down.sql new file mode 100644 index 000000000..5c3f8bde6 --- /dev/null +++ b/internal/storage/schema/migrations/0071_add_removed_restriction.down.sql @@ -0,0 +1,16 @@ +-- Reverse of 0071: drop issue_versions.removed_restriction. +-- +-- Guarded on INFORMATION_SCHEMA the same way 0067/0068's downs are, so a +-- partially-applied or already-rolled-back workspace rolls back safely. +-- Only migrations/*.up.sql is embedded into the CLI fresh bundle, so the +-- PREPARE hazard (cli_prepared_ddl.go) never reaches this file. +SET @issue_versions_rr_has = ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'removed_restriction' +); +SET @sql = IF(@issue_versions_rr_has > 0, + 'ALTER TABLE issue_versions DROP COLUMN removed_restriction', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/migrations/0071_add_removed_restriction.up.sql b/internal/storage/schema/migrations/0071_add_removed_restriction.up.sql new file mode 100644 index 000000000..e47123ed2 --- /dev/null +++ b/internal/storage/schema/migrations/0071_add_removed_restriction.up.sql @@ -0,0 +1,49 @@ +-- Migration 0071: R7.1 as-of read (gastownhall/beads#5898 revision 9, +-- gastownhall/beads#6136); conformance suite: backend/conformance/ +-- versioned_read_contract.go. +-- +-- issue_versions already carries removed_at DATETIME and removed_reason +-- VARCHAR(255) (migration 0067), virgin and unused until now: R7.1's +-- AsOfReadInTx (internal/storage/issueops/asof_read.go) reads them as the +-- durable "this version row was removed" marker on the resolved row, and +-- refuses a served answer whenever removed_at is set (R8: never silently +-- substitute a neighboring/surviving/current version). +-- +-- removed_restriction VARCHAR(30) is the one column this migration adds: the +-- categorical restriction (gone_retention / gone_erasure / +-- gone_reorganization / unknown) that removed_at's presence alone cannot +-- carry. Nullable and NULL for every existing row -- Live is the absence of +-- a value, never a stored one -- and populated only alongside removed_at, +-- never independently. 30 chars comfortably fits the longest local +-- constant (internal/storage/issueops/asof_read.go's AsOfRestriction, +-- "gone_reorganization", 19 bytes) with headroom, matching change_agent's +-- own no-tight-fit precedent elsewhere in this table. +-- +-- Forward-compatible with R17 (not yet implemented on this branch): R17's +-- eventual real hold/remove/erase primitives will be the production WRITER +-- of these three columns; R7.1 only reads them here. No future collision -- +-- this migration only adds the one column R17 will also need, it does not +-- claim the writing role. +-- +-- Guarded the same way 0067/0068 guard their ADD COLUMNs (see 0067's header +-- for the full explanation): no MariaDB-only IF NOT EXISTS on Dolt 2.2.3's +-- ADD COLUMN, so an INFORMATION_SCHEMA probe + PREPARE is the only +-- replay-safe shape. Needs a CLI-bundle direct-DDL override +-- (cliMigration0071AddRemovedRestriction in cli_migrations.go), the same +-- dolthub/dolt#11345 escape hatch 0067/0068 use, guarded by +-- TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified. +-- +-- No wisps twin: issue_versions has no wisps-side counterpart table (design +-- section 16.3), so cliSubstituteAssumesWispTables does not apply here +-- either, matching 0068. +SET @issue_versions_rr_needs_add = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'removed_restriction' +); +SET @sql = IF(@issue_versions_rr_needs_add = 1, + 'ALTER TABLE issue_versions ADD COLUMN removed_restriction VARCHAR(30)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index c1ffe2b30..b63c0b553 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -1974,6 +1974,9 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // 0070: two-plane prepared ADD COLUMN, same shape as 0067. "ALTER TABLE issues ADD COLUMN participation_generation BIGINT NULL;", "ALTER TABLE wisps ADD COLUMN participation_generation BIGINT NULL;", + // 0071: single-plane prepared ADD COLUMN, same shape as 0068's + // attribution_status (no wisps twin -- issue_versions has none). + "ALTER TABLE issue_versions ADD COLUMN removed_restriction VARCHAR(30);", } { if !strings.Contains(got, want) { t.Fatalf("AllMigrationsSQL missing direct CLI DDL %q", want) @@ -2010,6 +2013,9 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // 0070 guards both planes' ALTERs the same way, mirroring 0067's pair. "@issues_pg_needs_add", "@wisps_pg_needs_add", + // 0071 guards its ALTER the same way; only its source text carries + // this probe. + "@issue_versions_rr_needs_add", } { if strings.Contains(got, forbidden) { t.Fatalf("AllMigrationsSQL contains source prepared-DDL guard %q", forbidden) diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index ac9440eec..96c3d988c 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -175,6 +175,7 @@ go_test( "version_reconcile_boot_test.go", "version_reconcile_integration_test.go", "version_reconciler_contract_test.go", + "versioned_read_contract_test.go", "workspace_config_contract_test.go", ], data = [ diff --git a/internal/storage/uow/dualwrite_participation_generation_contract_test.go b/internal/storage/uow/dualwrite_participation_generation_contract_test.go index 4f2a585df..42cf07f8e 100644 --- a/internal/storage/uow/dualwrite_participation_generation_contract_test.go +++ b/internal/storage/uow/dualwrite_participation_generation_contract_test.go @@ -33,6 +33,9 @@ func TestParticipationGenerationFence(t *testing.T) { t.Run("ProceedsForPromotedRecord", func(t *testing.T) { conformance.RunParticipationGenerationProceedsForPromotedRecord(t, ctx, fixture) }) + t.Run("FlagOffSkipsUpdateOnParticipatingRecord", func(t *testing.T) { + conformance.RunParticipationGenerationFlagOffSkipsUpdateOnParticipatingRecord(t, ctx, fixture) + }) t.Run("FenceIgnoresSchemaSkewHatch", func(t *testing.T) { conformance.RunParticipationGenerationFenceIgnoresSchemaSkewHatch(t, ctx, fixture) }) diff --git a/internal/storage/uow/versioned_read_contract_test.go b/internal/storage/uow/versioned_read_contract_test.go new file mode 100644 index 000000000..947f2997f --- /dev/null +++ b/internal/storage/uow/versioned_read_contract_test.go @@ -0,0 +1,404 @@ +package uow + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/domain" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/types" +) + +// asOfReadHarness is this leg's Design D dispatch table (gastownhall/beads#5898 +// revision 9, gastownhall/beads#6136): the conformance suite +// invents storeID values to select among N genuinely isolated stores, and +// isolating them is entirely this harness's job -- AsOfReadInTx itself is +// storeID-oblivious (see the package doc on +// internal/storage/issueops/asof_read.go). Each storeID gets its own fresh +// UnitOfWorkProvider, built lazily on first use (provider) and torn down via +// t.Cleanup on the TestAsOfReadContract-level *testing.T. created tracks +// which (storeID, id) pairs already have a base issues row, since mintAt's +// second and later calls for the same pair must only add a historical +// issue_versions row, never a second base create. +// +// UNLIKE the dolt leg's buildAsOfReadStore (one shared testcontainer, N +// randomly-named databases) or the embeddeddolt leg's buildAsOfReadStore (N +// fully independent on-disk databases, isolation free), this leg spawns one +// real dolt sql-server SUBPROCESS per NewDoltServerUOWProvider call unless +// that call adopts an already-running one. Booting N subprocesses -- one per +// storeID, up to seven across a full TestAsOfReadContract run -- would be +// needlessly heavy, so this harness follows the same "one shared server, many +// databases" shape the dolt leg uses: TestAsOfReadContract does the ONE +// heavyweight, Fatal-permitted setup (dolt/bd binaries, HOME, port, +// storeRootDir, server config, log path) once in its own body, before any +// t.Run, and hands the resulting coordinates to newAsOfReadHarness. provider() +// then calls NewDoltServerUOWProvider -- a pure, Fatal-free function -- against +// those SAME coordinates with a fresh, randomly-named database per storeID. +// proxy.GetCreateDatabaseProxyServerEndpoint's adoption semantics (given the +// same rootDir, a later call dials the already-running server instead of +// spawning a duplicate, regardless of which database name it carries -- see +// TestNewDoltServerUOWProvider_ConcurrentInstantiation) mean only ONE server +// process is ever spawned, no matter how many storeIDs a case invents. +// +// storeID itself is used ONLY as a map key into h.providers here -- never as +// the database name. The conformance suite's storeID strings are built from +// free-form tags (asOfReadStore in backend/conformance/ +// versioned_read_contract.go: fixture.IssuePrefix + "-asof-store-" + tag, +// where tag can itself embed a Restriction's hyphenated String()), so they are +// not assumed to be safe SQL/Dolt identifiers. Generating an independent +// random per-store database name (asOfReadDatabaseName, matching the dolt +// leg's own dbSuffix approach) sidesteps that question entirely rather than +// sanitizing storeID. +// +// provider() cannot use newTestUOWProvider or newUOWRoleFixtureProvider. +// AsOfReadFixture's hook types (Resolve/MintAt/MakeUnanswerable) carry no +// *testing.T of their own, so a provider built lazily inside a hook -- itself +// invoked from inside a t.Run subtest closure, sometimes nested two deep (the +// restriction-loop case) -- has access only to whichever *testing.T this +// harness was constructed with: necessarily the OUTER TestAsOfReadContract's +// t, captured before any t.Run. Both of those helpers call testify's +// require.NoError / t.Fatalf internally, and calling FailNow/SkipNow on a +// *testing.T from a goroutine other than the one running that exact test -- +// which a t.Run subtest's closure always is, relative to the outer test -- +// panics with "subtest may have called FailNow on a parent test" (go's +// testing package requires FailNow/SkipNow to run only on the goroutine +// executing that specific test). provider() is a Fatalf/Skip-free path for +// exactly this reason: its failures come back as a plain error, which flows +// through the calling hook's own (..., error) return to +// conformance.RunAsOfReadXXX, which correctly Fatalfs using the SUBTEST's own +// *testing.T on the SUBTEST's own goroutine. Also unlike +// newUOWRoleFixtureProvider (one provider for a whole role suite, matching +// roleFixtureKit's frozen shape), this harness needs N independently-isolated +// providers -- roleFixtureKit is deliberately not reused here; see this +// file's own note by conformanceToAsOfRestriction on why issueops must stay +// conformance-oblivious, the same boundary that keeps this harness's own +// mapping from living centrally. +// +// See internal/storage/dolt/versioned_read_contract_test.go and +// internal/storage/embeddeddolt/versioned_read_contract_test.go for this +// leg's siblings, which this file mirrors as closely as the three backends' +// construction APIs allow. +type asOfReadHarness struct { + t *testing.T + storeRootDir string + doltBin string + cfgPath string + logPath string + providers map[string]UnitOfWorkProvider + created map[[2]string]bool +} + +func newAsOfReadHarness(t *testing.T, storeRootDir, doltBin, cfgPath, logPath string) *asOfReadHarness { + return &asOfReadHarness{ + t: t, + storeRootDir: storeRootDir, + doltBin: doltBin, + cfgPath: cfgPath, + logPath: logPath, + providers: map[string]UnitOfWorkProvider{}, + created: map[[2]string]bool{}, + } +} + +// provider returns storeID's UnitOfWorkProvider, building it on first use +// against a fresh, randomly-named database on the shared server. See the +// asOfReadHarness doc comment for why this function -- and everything it +// calls -- must never call *testing.T's Fatalf/Skip. +func (h *asOfReadHarness) provider(ctx context.Context, storeID string) (UnitOfWorkProvider, error) { + if p, ok := h.providers[storeID]; ok { + return p, nil + } + database, err := asOfReadDatabaseName() + if err != nil { + return nil, err + } + provider, err := NewDoltServerUOWProvider( + ctx, + h.storeRootDir, + database, + h.logPath, + h.cfgPath, + proxy.BackendLocalServer, + "root", + "", + h.doltBin, + 0, + 0, + false, + "", + ) + if err != nil { + return nil, fmt.Errorf("as-of read fixture: new provider for store %q (database %q): %w", storeID, database, err) + } + // t.Cleanup only appends to a mutex-guarded slice -- unlike Fatalf/Skip it + // never calls runtime.Goexit, so registering it on the outer h.t from + // inside a subtest's goroutine is safe. h.t's own cleanups run only after + // ALL of TestAsOfReadContract's t.Run calls have returned (none of them, + // nor their own nested t.Run calls, ever invoke t.Parallel), so every + // provider this harness builds stays open for the whole test. These + // per-provider Close cleanups are registered AFTER (so, per t.Cleanup's + // LIFO order, run BEFORE) TestAsOfReadContract's own shared + // verifiedShutdownCleanup (proxy.Shutdown) cleanup, so every provider is + // closed while the server they share is still up. + h.t.Cleanup(func() { _ = provider.Close(context.Background()) }) + + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "bd: set issue prefix", uw.ConfigUseCase().SetConfig(ctx, "issue_prefix", "asof") + }); err != nil { + return nil, fmt.Errorf("as-of read fixture: set issue_prefix for store %q: %w", storeID, err) + } + + h.providers[storeID] = provider + return provider, nil +} + +// asOfReadDatabaseName mints a fresh, unique database name, independent of +// any storeID's own content -- see the asOfReadHarness doc comment for why +// storeID itself is never used as one. +func asOfReadDatabaseName() (string, error) { + suffix := make([]byte, 6) + if _, err := rand.Read(suffix); err != nil { + return "", fmt.Errorf("as-of read fixture: generate db name: %w", err) + } + return "asof_" + hex.EncodeToString(suffix), nil +} + +// ensureCreated bare-creates id in storeID's store the first time it is +// asked for, giving RecordVersionAtInTx (called from mintAt) a base issues +// row to snapshot -- recordVersionAtInTx starts with GetIssueInTx, so it has +// nothing to read until this has run once. This goes through the real +// IssueUseCase, the same production entry point batch_creator.go and +// issue_operations.go use, unlike resolve/mintAt/makeUnanswerable below, +// which reach the shared issueops body directly (see the package doc on +// internal/storage/issueops/asof_read.go: "ALL THREE LEGS SHARE THIS BODY"). +func (h *asOfReadHarness) ensureCreated(ctx context.Context, storeID, id string) error { + key := [2]string{storeID, id} + if h.created[key] { + return nil + } + provider, err := h.provider(ctx, storeID) + if err != nil { + return err + } + issue := &types.Issue{ + ID: id, + Title: "as-of read fixture issue", + Status: types.StatusOpen, + Priority: 3, + IssueType: types.TypeTask, + } + err = RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + params := domain.CreateIssueParams{ + Issue: issue, + ExplicitID: issue.ID, + CreateOnly: true, + } + _, err := uw.IssueUseCase().CreateIssue(ctx, params, "asof-read-fixture") + return "as-of read fixture: bare-create " + issue.ID, err + }) + if err != nil { + return fmt.Errorf("as-of read fixture: bare-create %s: %w", id, err) + } + h.created[key] = true + return nil +} + +func (h *asOfReadHarness) resolve(ctx context.Context, storeID, id string, selector conformance.AsOfSelector) (conformance.AsOfReadResult, error) { + provider, err := h.provider(ctx, storeID) + if err != nil { + return conformance.AsOfReadResult{}, err + } + result, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (storageissueops.AsOfReadResult, error) { + runner, err := importStatementRunner(uw) + if err != nil { + return storageissueops.AsOfReadResult{}, err + } + return storageissueops.AsOfReadInTx(ctx, runner, storeID, id, toIssueopsSelector(selector)) + }) + if err != nil { + return conformance.AsOfReadResult{}, err + } + return toConformanceResult(result), nil +} + +// mintAt is the two-phase implementation MintAt's contract needs: phase one +// (ensureCreated) runs only once per (storeID, id); phase two +// (RecordVersionAtInTx) runs every call and always supplies the row mintAt +// reports the address of. RecordVersionAtInTx deliberately does not gate on +// versionedHistoryEnabled (this harness never turns that on for its stores), +// so it always mints, and reading MAX(revision) back inside the same +// transaction (read-your-writes) is how we learn which revision it picked -- +// RecordVersionAtInTx itself reports no revision number. +// +// The commit message returned to RunTxResult is deliberately non-empty: an +// empty commitMsg makes RunTxResultWithin skip calling uw.Commit entirely +// (tx.go), so the write would never persist past this attempt's implicit +// rollback -- exactly the RunTxEphemeral hazard its own doc comment warns +// against for versioned tables. +func (h *asOfReadHarness) mintAt(ctx context.Context, storeID, id string, acceptedAt time.Time) (conformance.Address, error) { + if err := h.ensureCreated(ctx, storeID, id); err != nil { + return "", err + } + provider, err := h.provider(ctx, storeID) + if err != nil { + return "", err + } + revision, err := RunTxResult(ctx, provider, func(ctx context.Context, uw UnitOfWork) (int64, string, error) { + runner, err := importStatementRunner(uw) + if err != nil { + return 0, "", err + } + if err := storageissueops.RecordVersionAtInTx(ctx, runner, id, "asof-read-fixture", acceptedAt); err != nil { + return 0, "", err + } + var rev int64 + if err := runner.QueryRowContext(ctx, + `SELECT MAX(revision) FROM issue_versions WHERE issue_id = ?`, id, + ).Scan(&rev); err != nil { + return 0, "", err + } + return rev, "as-of read fixture: mint version for " + id, nil + }) + if err != nil { + return "", err + } + return conformance.Address(storageissueops.AsOfVersionAddress(id, revision)), nil +} + +func (h *asOfReadHarness) makeUnanswerable(ctx context.Context, storeID, id string, restriction conformance.Restriction) error { + asOfRestriction, ok := conformanceToAsOfRestriction[restriction] + if !ok { + return fmt.Errorf("as-of read fixture: unmapped restriction %v", restriction) + } + provider, err := h.provider(ctx, storeID) + if err != nil { + return err + } + reason := fmt.Sprintf("as-of read fixture: made unanswerable as %s", restriction) + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + runner, err := importStatementRunner(uw) + if err != nil { + return "", err + } + if err := storageissueops.MarkLatestVersionRemovedInTx(ctx, runner, id, asOfRestriction, reason); err != nil { + return "", err + } + return "as-of read fixture: mark " + id + " removed", nil + }) +} + +// conformanceToAsOfRestriction and its inverse are this leg's translation +// boundary between backend/conformance.Restriction (hyphenated String(), +// int-backed) and issueops.AsOfRestriction (underscored, string-backed -- +// its values ARE the removed_restriction column's contents). issueops must +// stay conformance-oblivious and conformance must stay backend-oblivious +// (internal/storage/issueops/asof_read.go's package doc), so this mapping is +// deliberately duplicated per leg rather than shared centrally. +var conformanceToAsOfRestriction = map[conformance.Restriction]storageissueops.AsOfRestriction{ + conformance.RestrictionLive: storageissueops.AsOfRestrictionLive, + conformance.RestrictionGoneRetention: storageissueops.AsOfRestrictionGoneRetention, + conformance.RestrictionGoneErasure: storageissueops.AsOfRestrictionGoneErasure, + conformance.RestrictionGoneReorganization: storageissueops.AsOfRestrictionGoneReorganization, + conformance.RestrictionUnknown: storageissueops.AsOfRestrictionUnknown, +} + +var asOfRestrictionToConformance = map[storageissueops.AsOfRestriction]conformance.Restriction{ + storageissueops.AsOfRestrictionLive: conformance.RestrictionLive, + storageissueops.AsOfRestrictionGoneRetention: conformance.RestrictionGoneRetention, + storageissueops.AsOfRestrictionGoneErasure: conformance.RestrictionGoneErasure, + storageissueops.AsOfRestrictionGoneReorganization: conformance.RestrictionGoneReorganization, + storageissueops.AsOfRestrictionUnknown: conformance.RestrictionUnknown, +} + +func toIssueopsSelector(selector conformance.AsOfSelector) storageissueops.AsOfSelector { + out := storageissueops.AsOfSelector{At: selector.At} + if selector.Version != nil { + out.Address = string(*selector.Version) + } + return out +} + +func toConformanceResult(result storageissueops.AsOfReadResult) conformance.AsOfReadResult { + if result.Refused { + return conformance.AsOfReadResult{ + Refusal: &conformance.AsOfReadRefusal{ + Restriction: asOfRestrictionToConformance[result.Restriction], + Reason: result.Reason, + }, + } + } + return conformance.AsOfReadResult{ + Address: conformance.Address(result.Address), + State: result.State, + } +} + +// TestAsOfReadContract wires this leg into the R7.1 as-of-read contract +// (gastownhall/beads#5898 revision 9, gastownhall/beads#6136) with a real +// Resolve/MintAt/MakeUnanswerable backed by one isolated UnitOfWorkProvider +// per storeID (asOfReadHarness). +func TestAsOfReadContract(t *testing.T) { + // Everything through logPath below is the same one-time, Fatal-permitted + // setup newTestUOWProvider does, run here directly instead of through + // that helper: newTestUOWProvider both sets up AND builds one provider + // bound to a fixed "beads" database, but this harness needs the setup + // coordinates kept separate so provider() can build N providers, one per + // storeID, lazily from inside subtest closures. See the asOfReadHarness + // doc comment for why provider() must stay Fatalf/Skip-free, which is + // exactly why this block -- the one place in this file where Fatalf/Skip + // is safe -- must run here, before any t.Run. + testutil.RequireDoltBinary(t) + bin, err := exec.LookPath("dolt") + require.NoError(t, err) + + bdBin := buildBDBinary(t) + prev := proxy.ResolveExecutable + proxy.ResolveExecutable = func() (string, error) { return bdBin, nil } + t.Cleanup(func() { proxy.ResolveExecutable = prev }) + + t.Setenv("HOME", t.TempDir()) + + port, err := proxy.PickFreePort() + require.NoError(t, err) + storeRootDir := t.TempDir() + shutdownOnInterrupt(t, storeRootDir) + verifiedShutdownCleanup(t, storeRootDir) + cfgPath := writeServerConfig(t, port) + logPath := filepath.Join(t.TempDir(), "server.log") + + ctx := context.Background() + h := newAsOfReadHarness(t, storeRootDir, bin, cfgPath, logPath) + fixture := conformance.AsOfReadFixture{ + IssuePrefix: "asof", + Resolve: h.resolve, + MintAt: h.mintAt, + MakeUnanswerable: h.makeUnanswerable, + } + + t.Run("SelectorAcceptsEitherAnInstantOrAVersionAddress", func(t *testing.T) { + conformance.RunAsOfReadSelectorAcceptsEitherAnInstantOrAVersionAddress(t, ctx, fixture) + }) + t.Run("ReturnsTheLatestVersionAtOrBeforeTStrictlyExcludingLater", func(t *testing.T) { + conformance.RunAsOfReadReturnsTheLatestVersionAtOrBeforeTStrictlyExcludingLater(t, ctx, fixture) + }) + t.Run("ReportsTheAddressItServed", func(t *testing.T) { + conformance.RunAsOfReadReportsTheAddressItServed(t, ctx, fixture) + }) + t.Run("RefusesAsGoneOrUnknownNeverSubstitutingAnotherVersion", func(t *testing.T) { + conformance.RunAsOfReadRefusesAsGoneOrUnknownNeverSubstitutingAnotherVersion(t, ctx, fixture) + }) + t.Run("IsCompleteOnlyForTheAnsweringStoresHoldings", func(t *testing.T) { + conformance.RunAsOfReadIsCompleteOnlyForTheAnsweringStoresHoldings(t, ctx, fixture) + }) +} diff --git a/internal/storage/version_lister.go b/internal/storage/version_lister.go new file mode 100644 index 000000000..df0946780 --- /dev/null +++ b/internal/storage/version_lister.go @@ -0,0 +1,76 @@ +package storage + +import ( + "context" + "time" +) + +// IssueVersion is one accepted, immutable durable state of one issue, as +// issue_versions records it. +// +// Revision is a PER-STORE ORDINAL, not a portable address: two disconnected +// clones can both hold revision 8 for the same issue (migration 0067's own +// header says so, and the flag is single-writer-only until the version_id +// primary key lands). Callers must not present it as a citable address. +// +// Its JSON key is "local_revision", NOT "revision", for two reasons that +// compound. types.Issue already ships `json:"revision"` (types.go:1206) and +// that is the row-lock CAS token, an unrelated thing -- and it is a STRING +// there while this is an int64, so a script that learned one shape breaks on +// the other. The name also carries the non-citability with the value, where a +// machine reader sees it, instead of only in a footer no script parses. It +// needs no rename when version_id lands beside it. (bee-ghosttrack, #5898 +// consumer read, 2026-09-20.) +// +// ChangeAgent is read from issue_versions.change_agent (migration 0067) but +// is deliberately NOT part of the JSON shape, which is what `bd versions +// --json` prints. Provenance V2 (gastownhall/beads#7510) replaces the ad hoc +// agent field with an actor chain, so publishing it now would make a public +// contract out of a field that is on its way out. The column and the field +// stay; only the JSON key is withheld. +// +// It lives here rather than in issueops for the same reason HistoryEntry +// does: issueops imports storage, so the shared row type has to sit on this +// side of that edge. +type IssueVersion struct { + IssueID string `json:"issue_id"` + Revision int64 `json:"local_revision"` + Epoch int64 `json:"epoch"` + ChangeActor string `json:"change_actor"` + ChangeAgent string `json:"-"` + ChangeMessage string `json:"change_message"` + ChangeAt time.Time `json:"change_at"` + AttributionStatus string `json:"attribution_status"` + RemovedAt *time.Time `json:"removed_at"` + RemovedReason string `json:"removed_reason"` + RemovedRestriction string `json:"removed_restriction"` + StateBytes int64 `json:"state_bytes"` +} + +// Removed reports whether this version is no longer served -- retention, +// erasure or an epoch reorganization removed it. A removed version keeps its +// row (so the gap is legible) but its durable state is gone, and callers must +// render it as a restriction rather than as an empty version. +func (v IssueVersion) Removed() bool { return v.RemovedAt != nil } + +// VersionLister is implemented by backends that can serve an issue's +// versioned history from the issue_versions table (migration 0067+). +// +// This is a narrow capability rather than a method on Storage on purpose: a +// backend that has no issue_versions table -- a proxied client, a no-db +// workspace, a non-Dolt backend -- should not be forced to implement it, and +// a caller that asserts for it learns the truth instead of receiving a stub +// that answers "no versions" for a store that simply cannot know. Same +// reasoning as ExternalRefHistoryQuerier in history_viewer.go. +// +// Holding this capability says the backend can READ versions. It says nothing +// about whether version recording is switched on -- that is +// VersionedHistoryConfigurer, and a store can legitimately implement both +// while the feature is off, in which case ListVersions truthfully returns +// nothing. +type VersionLister interface { + // ListVersions returns every retained version of issueID, newest first. + // An empty result means no retained versions, never "unsupported" -- + // unsupported is the failed type assertion. + ListVersions(ctx context.Context, issueID string) ([]IssueVersion, error) +} diff --git a/internal/versionedhistory/BUILD.bazel b/internal/versionedhistory/BUILD.bazel new file mode 100644 index 000000000..250b9fd91 --- /dev/null +++ b/internal/versionedhistory/BUILD.bazel @@ -0,0 +1,92 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "versionedhistory", + srcs = ["activation.go"], + importpath = "github.com/steveyegge/beads/internal/versionedhistory", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/debug", + "//internal/storage", + "//internal/storage/uow", + "//issueops", + ], +) + +go_test( + name = "versionedhistory_test", + srcs = ["activation_test.go"], + embed = [":versionedhistory"], + deps = [ + "//internal/config", + "//internal/debug", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/externaldeps", + "//internal/storage/uow", + "//internal/telemetry", + "//issueops", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/internal/versionedhistory/activation.go b/internal/versionedhistory/activation.go new file mode 100644 index 000000000..d187b2758 --- /dev/null +++ b/internal/versionedhistory/activation.go @@ -0,0 +1,350 @@ +// Package versionedhistory decides whether dual-write issue-version history is on +// for one store or unit-of-work provider, and applies that decision where the +// thing is constructed. +// +// It is a package rather than helpers inside cmd/bd for the same reason as +// internal/eventsjournal: bd doctor's repair handlers live in their own package +// and cannot import package main, and a repair that mutates issues needs exactly +// the same rule as the command surface. There is ONE implementation of that rule, +// here. +// +// THE SWITCH IS A PROPERTY OF THE STORE. `bd config set versioned-history.enabled +// true` writes a row of the workspace's own config table, and a store answers for +// itself: a write routed into rig B reads rig B's row, never the launching +// workspace's. The environment (BD_VERSIONED_HISTORY_ENABLED) and a hand-edited +// config.yaml value are read through viper, are process-wide, and can only turn +// recording ON for every store the process opens; nothing but the row turns a +// store off. The two are OR'd rather than ranked, deliberately: the hazard is a +// write that fails to record, never one that records when it needn't, so either +// plane saying yes is enough and neither can silently switch the other off. +// +// Activation is applied by the FACTORIES that construct a store or provider +// (ActivateStore, ActivateProvider), never by the commands that use one. Applying +// it at the root pre-run reached only the store a command opens for its own +// workspace, and missed every store a routed update, a routed close, `bd create +// --repo` or the direct-mode open builds for another one -- which then recorded no +// history while the command reported success. See +// cmd/bd/versioned_history_construction_test.go for the structural guard. +package versionedhistory + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/issueops" +) + +// ConfigKey is the one spelling of the switch, used for the store-config read +// below and for the `bd config set` line the help text and the off-refusal tell +// people to run. They must not drift. +const ConfigKey = "versioned-history.enabled" + +// Enabled reports whether version recording is on for this store, reading BOTH +// planes bd stores configuration in. +// +// This exists because the two disagreed. `bd config set versioned-history.enabled +// true` -- the command the help text and the off-refusal both name -- writes the +// workspace DATABASE, because the key is not a yaml-only key (config.IsYamlOnlyKey, +// and "versioned-history." is not among the prefixes at yaml_config.go). +// config.GetBool reads viper, which is yaml and env only. So the advertised command +// wrote one plane and the reader read the other: the user ran exactly what they +// were told, was told it worked, and the feature stayed off. Configured on, +// actually off -- the GH#536 class, and the precise failure this command was added +// to end. (bee-ghosttrack, #6661 review, finding 1.) +// +// The store plane is authoritative in the sense that matters: it makes the switch +// a property of the STORE rather than of whoever is calling, so two clients of one +// dolt sql-server agree about whether a write records a version. That closes the +// silent-holes case in the same review's finding 2 -- a client with recording off +// advances nothing, and the next enabled write mints a version that absorbs the +// unrecorded change under its own actor, producing contiguous revisions and a +// gapless-LOOKING history that is not one. +// +// The two are OR'd rather than ranked, deliberately. The hazard is a write that +// fails to record, never one that records when it needn't, so either plane saying +// yes is enough and neither can silently switch the other off. That keeps +// BD_VERSIONED_HISTORY_ENABLED=1 working for a one-off run without letting an unset +// (or =0) env var countermand a store that is recording. +// +// A store that cannot answer is not an error: proxied and no-db workspaces have no +// settings plane to read. Those fall back to viper alone rather than failing the +// command, because a store-config read is not worth breaking every bd invocation +// over. +// +// Nothing in production calls Enabled today. `bd versions` reports StoreSetting +// alone, because it only reads: the process-wide planes cannot make a store +// record, so they cannot speak for it. Activation uses EnabledForStore. Enabled +// stays, with its tests, as the ungated form of the combined rule. +func Enabled(ctx context.Context, st storage.DoltStorage) bool { + if config.GetBool(ConfigKey) { + // Already on via env or yaml; skip the store read entirely. This is also + // what keeps the common path cheap when someone is driving the feature from + // the environment. + return true + } + return StoreSetting(ctx, st) +} + +// StoreSetting reads the switch from the store's own settings plane, answering +// false for every reason a read can fail to produce a true. +// +// "EVERY reason" INCLUDES A PANIC, and that is the whole point of the defer below +// rather than an oversight to tidy away. The factories call this on every store +// they hand back, before anything has established that the store at the bottom of +// the chain can answer a config question at all -- so a store that cannot is not a +// hypothetical, it is a store that takes the process down. The shape that found +// it: a test double embedding a nil storage.DoltStorage, whose promoted +// WorkspaceConfig SIGSEGVs on call. The blast radius of letting that through is not +// one command; it is every bd invocation against any store whose settings plane is +// absent or half-built, for a read whose answer only ever decides whether to record +// versions. (bee-ghosttrack, #6661 second review.) +// +// A CAPABILITY ASSERTION ON WorkspaceConfig CANNOT REPLACE THIS, because +// WorkspaceConfig is declared on storage.DoltStorage itself +// (internal/storage/storage.go): every store satisfies an +// interface{ WorkspaceConfig() ... } assertion, including the one that panics when +// the method is actually called. That assertion would pass and the panic would +// happen anyway. +// +// AN ASSERTION ON storage.VersionedHistoryConfigurer IS DIFFERENT, and it is the +// primary guard now -- see EnabledForStore below. SetVersionedHistoryEnabled is NOT +// part of DoltStorage, so a store that embeds a nil DoltStorage does not promote it +// and does not satisfy that interface. (bee-ghosttrack, #6661 third review.) +// +// The recover stays anyway, because the guard does not cover every caller: `bd +// versions` calls StoreSetting directly, where the read is the point and no +// capability gate precedes it, and because a store that DOES implement the +// capability can still have a half-built settings plane. Guard first, recover as +// the backstop. +// +// A typed nil needs no separate guard for the same reason it needs no reflection: +// if its WorkspaceConfig panics, the recover answers false; if it returns (nil, nil) +// instead, the cfg == nil check answers false. Both arms are already here. +// +// The answer when the store cannot be read is false -- recording off, which is what +// the feature ships as. That is a real tradeoff and not a free one: a store that IS +// recording, whose settings read panics, is read as not recording, and a write +// during that window advances nothing. It is still the right side to fail to, +// because the alternative is that bd does not run at all, and a store too broken to +// answer a settings query is not one that was successfully recording versions a +// moment ago. +// +// WHAT IS AND IS NOT DIAGNOSED. An absent row is "" with a nil error by contract +// (WorkspaceConfig.GetSetting) and is the normal state of every workspace that never +// ran `bd config set`: it stays silently off. A read that ERRORS is different -- the +// row may well be there -- and resolving it to off with nothing to say so is the +// "set but silently off" shape this feature exists to end. Those go to debug.Logf, +// not to stderr: this runs on every invocation, and a diagnostic on a path the user +// did not ask about would be noise on all of them. +func StoreSetting(ctx context.Context, st storage.DoltStorage) (enabled bool) { + if st == nil { + return false + } + defer func() { + if r := recover(); r != nil { + // Deliberately swallowed, not re-raised and not logged: this runs on + // every invocation and its answer is a default, so the honest report is + // the false below rather than noise on a path the user did not ask + // about. The named return makes that explicit. + // + // This is the opposite choice from the repo's other non-test recover + // (internal/storage/dolt/transaction.go), which rolls back and then + // re-panics, and the difference is not stylistic: there a caller must + // not proceed over a half-applied transaction, so the panic has to keep + // traveling. Here the caller can proceed perfectly well, because what it + // wanted was a boolean with a documented default. + enabled = false + } + }() + cfg, err := st.WorkspaceConfig() + if err != nil { + debug.Logf("versioned history: %s left off: no settings plane readable on %T: %v\n", ConfigKey, st, err) + return false + } + if cfg == nil { + return false + } + res, err := cfg.GetSetting(ctx, issueops.GetSettingRequest{Key: ConfigKey}) + if err != nil { + debug.Logf("versioned history: %s left off: reading the setting from %T failed: %v\n", ConfigKey, st, err) + return false + } + return parseSettingBool(res.Value) +} + +// parseSettingBool reads the stored string the way `bd config set` writes it. +// Unset is "" and a nil error by contract (WorkspaceConfig.GetSetting), which lands +// here as false -- the correct default for a flag that ships off. +func parseSettingBool(v string) bool { + b, err := strconv.ParseBool(strings.TrimSpace(v)) + return err == nil && b +} + +// ValueEnables reports whether a stored value of the switch turns recording on, +// reading it the way every reader of the row does. `bd config set` uses it to +// decide whether the value it was just given deserves the replication and +// single-writer caveat, so that answer cannot drift from what the stores do with +// the same string. +func ValueEnables(v string) bool { return parseSettingBool(v) } + +// EnabledForStore answers the switch for a freshly constructed store, which is the +// caller that runs on EVERY store bd opens. +// +// It differs from Enabled in one deliberate way: it will not read the store's +// settings plane unless the store could actually act on the answer. Activation's +// whole use for the result is Apply, which needs a storage.VersionedHistoryConfigurer; +// a store that is not one cannot record versions however the question is answered, +// so the read is pure cost and, for a store whose settings plane is absent or +// half-built, pure risk. This is what took CI red at 32d4f966e: the read panicked at +// wiring time on a test double embedding a nil storage.DoltStorage, which is a panic +// in a path whose answer only ever decides whether to record versions. +// +// The ordering matters and is not interchangeable with Apply's own assertion. The +// viper plane is read FIRST and unconditionally, so a store that is +// set-but-unsupported still reaches Apply with enabled=true and still gets the +// warning it prints. Only the STORE-plane read is gated. The one case that loses its +// warning is a store whose own settings plane says true but which lacks the +// capability -- and `bd config set` could not have written that plane on such a +// store in the first place. +// +// (bee-ghosttrack, #6661 third review, blocking finding 1.) +func EnabledForStore(ctx context.Context, st storage.DoltStorage) bool { + if config.GetBool(ConfigKey) { + return true + } + if _, canRecord := st.(storage.VersionedHistoryConfigurer); !canRecord { + return false + } + return StoreSetting(ctx, st) +} + +// ProviderSetting reads the switch from a unit-of-work provider's own database, +// answering false for every reason a read can fail to produce a true. It is the +// provider-path twin of StoreSetting: a provider has no storage.DoltStorage to ask +// for a settings plane, so the row is read the way the rest of the unit-of-work +// layer reads one -- through a unit of work's ConfigUseCase, as +// internal/storage/uow/issue_operations.go does for claim.pools. +// +// The read opens one unit of work, asks for one config row, and rolls it back: a +// pooled connection, START TRANSACTION, a SELECT, a ROLLBACK. It starts no server +// and creates no schema (the provider constructors have already opened the pool and +// run initSchema by the time this is called) and commits nothing. That is the whole +// cost of "each provider answers for its own database", and it is why this is read +// at construction, once, rather than falling back to viper alone: a provider that +// silently ignored the row would record nothing for a workspace that asked, while +// every command reported success. +// +// The same failure policy as StoreSetting applies, including the recover. +func ProviderSetting(ctx context.Context, p uow.UnitOfWorkProvider) (enabled bool) { + if p == nil { + return false + } + defer func() { + if r := recover(); r != nil { + enabled = false + } + }() + unit, err := p.NewUOW(ctx) + if err != nil { + debug.Logf("versioned history: %s left off: opening a unit of work on %T to read it failed: %v\n", ConfigKey, p, err) + return false + } + // Rolled back, never committed: this is a read. + defer unit.Close(ctx) + raw, err := unit.ConfigUseCase().GetConfig(ctx, ConfigKey) + if err != nil { + debug.Logf("versioned history: %s left off: reading the setting through %T failed: %v\n", ConfigKey, p, err) + return false + } + return parseSettingBool(raw) +} + +// EnabledForProvider is EnabledForStore for a provider: viper first, then the +// capability check, then the provider's own row. +func EnabledForProvider(ctx context.Context, p uow.UnitOfWorkProvider) bool { + if config.GetBool(ConfigKey) { + return true + } + if _, canRecord := p.(storage.VersionedHistoryConfigurer); !canRecord { + return false + } + return ProviderSetting(ctx, p) +} + +// ActivateStore applies a freshly opened store's own configured switch to it. It is +// written to be used as a deferred rewrite of a factory's named results: +// +// func newX(ctx context.Context, ...) (s storage.DoltStorage, err error) { +// defer func() { s, err = versionedhistory.ActivateStore(ctx, s, err) }() +// ... open and return ... +// } +// +// The shape is deliberate, and is internal/eventsjournal's. It keeps the open and +// its activation in ONE function body, so "constructs a store" and "activates +// history on it" are the same syntactic unit -- which is what lets the construction +// guard check the property structurally instead of chasing a call graph. +// +// The store is activated RAW: every factory hands back the concrete store, before +// any decorator wraps it. That ordering is load-bearing, because activation is a +// type assertion and none of the decorators (telemetry, externaldeps, hooks) +// forwards SetVersionedHistoryEnabled -- asserting against a wrapper would fail +// silently and leave history off while the config said it was on. It is pinned by +// TestApplyReachesOnlyTheRawStore. +// +// A failed open passes through untouched, so the typed-nil store never reaches the +// activation. A store that cannot version at all is returned as-is without its +// settings plane being read: there is nothing to activate, and the factories run for +// every command that opens a store. +func ActivateStore(ctx context.Context, s storage.DoltStorage, err error) (storage.DoltStorage, error) { + if err != nil || s == nil { + return s, err + } + Apply(s, EnabledForStore(ctx, s)) + return s, nil +} + +// ActivateProvider is ActivateStore for a unit-of-work provider -- bd's second write +// plumbing, with its own transactions and its own activation switch, which +// proxied-server mode and `bd serve` against a server-mode workspace write through +// instead of a DoltStorage. +func ActivateProvider(ctx context.Context, p uow.UnitOfWorkProvider, err error) (uow.UnitOfWorkProvider, error) { + if err != nil || p == nil { + return p, err + } + Apply(p, EnabledForProvider(ctx, p)) + return p, nil +} + +// Apply binds a resolved answer to one plumbing -- a store or a unit-of-work +// provider. Activation is per instance rather than process-global: a process can +// hold several plumbings at once (a routed create holds two), and enabling history +// on one must not enable it on the rest. +// +// It only ever turns recording ON. A false answer touches nothing, so flag-off is +// byte-identical to a build without the feature, which is the contract #6135 ships +// under. +// +// A plumbing that does not implement the capability is not an error: proxied and +// no-db backends legitimately do not. But it is not silently ignored either -- +// asking for history and not getting it is exactly the case a user must be told +// about, so it warns. +func Apply(target any, enabled bool) { + if !enabled { + return + } + configurer, ok := target.(storage.VersionedHistoryConfigurer) + if !ok { + fmt.Fprintf(os.Stderr, + "warning: versioned-history.enabled is set, but this storage backend (%T) does not support version history; it stays off\n", + target) + return + } + configurer.SetVersionedHistoryEnabled(true) +} diff --git a/internal/versionedhistory/activation_test.go b/internal/versionedhistory/activation_test.go new file mode 100644 index 000000000..a043f6b16 --- /dev/null +++ b/internal/versionedhistory/activation_test.go @@ -0,0 +1,776 @@ +package versionedhistory + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/externaldeps" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/telemetry" + "github.com/steveyegge/beads/issueops" +) + +// The resolver's tests. Most of them moved here from cmd/bd with the code they +// test, assertions unchanged: that is what shows the move preserved #6661's +// behavior. The rest are the new claims -- each store (and each provider) answers +// for itself, the environment is process-wide and can only turn recording on, and +// a read that errors is not the same as a row that is absent. + +const envVar = "BD_VERSIONED_HISTORY_ENABLED" + +// viperEnv sets the switch's environment variable and re-initialises viper, since +// config.GetBool answers false until viper is initialised and so would make every +// env-driven case below pass whatever the code did with the env. +func viperEnv(t *testing.T, value string) { + t.Helper() + t.Setenv(envVar, value) + // Nothing from the machine's own config may decide these cases. + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Chdir(t.TempDir()) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize(): %v", err) + } + t.Cleanup(config.ResetForTesting) +} + +// captureStderr runs fn with os.Stderr redirected and returns what it wrote. With +// verbose set, debug.Logf writes too. +func captureStderr(t *testing.T, verbose bool, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + orig := os.Stderr + os.Stderr = w + debug.SetVerbose(verbose) + func() { + defer func() { + debug.SetVerbose(false) + os.Stderr = orig + _ = w.Close() + }() + fn() + }() + out, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return string(out) +} + +// --- fakes ----------------------------------------------------------------- + +// settingsPlane is a workspace settings plane with one canned answer. +type settingsPlane struct { + issueops.WorkspaceConfig + value string + err error + panicMsg string + reads int +} + +func (p *settingsPlane) GetSetting(_ context.Context, req issueops.GetSettingRequest) (issueops.SettingResult, error) { + p.reads++ + if p.panicMsg != "" { + panic(p.panicMsg) + } + return issueops.SettingResult{Key: req.Key, Value: p.value}, p.err +} + +// configProbeStore is a raw store at the bottom of a decorator chain whose +// settings plane behaves however a case needs: panicking, erroring, absent, or +// answering. It counts the reads so a test can prove it was actually asked, rather +// than passing because the switch short-circuited before reaching it. +// +// Every other method promotes from the embedded nil interface and panics if +// called, which is deliberate: nothing here should be calling them. +type configProbeStore struct { + storage.DoltStorage + cfg issueops.WorkspaceConfig + err error + panicMsg string + calls int + enabled bool + sets int +} + +func (s *configProbeStore) WorkspaceConfig() (issueops.WorkspaceConfig, error) { + s.calls++ + if s.panicMsg != "" { + panic(s.panicMsg) + } + return s.cfg, s.err +} + +// SetVersionedHistoryEnabled makes this store a storage.VersionedHistoryConfigurer, +// which is what earns it the store-plane read at activation time. Without it this +// store would be skipped before WorkspaceConfig was ever called, and every +// calls-count assertion below would pass for the wrong reason. +func (s *configProbeStore) SetVersionedHistoryEnabled(enabled bool) { + s.enabled = enabled + s.sets++ +} + +// nonConfigurerStore is the shape the capability guard exists for, and it is not a +// hypothetical: it is cmd/bd's own fakeProbingDoltStore, which embeds a nil +// storage.DoltStorage and documents that only its own methods may be called. +// SetVersionedHistoryEnabled is not part of DoltStorage, so nothing promotes it and +// this type does not satisfy storage.VersionedHistoryConfigurer -- while +// WorkspaceConfig IS part of DoltStorage, so it promotes, type-checks, and SIGSEGVs +// on call. +type nonConfigurerStore struct { + storage.DoltStorage + calls int +} + +func (s *nonConfigurerStore) WorkspaceConfig() (issueops.WorkspaceConfig, error) { + s.calls++ + panic("WorkspaceConfig on a store with no settings plane") +} + +// planeOnlyStore can answer a settings query but cannot record versions, which is +// what `bd versions` may be pointed at: the read is the point there. +type planeOnlyStore struct { + storage.DoltStorage + cfg issueops.WorkspaceConfig + calls int +} + +func (s *planeOnlyStore) WorkspaceConfig() (issueops.WorkspaceConfig, error) { + s.calls++ + return s.cfg, nil +} + +// nilDerefStore reproduces the exact shape that broke CI: the method body touches +// the receiver, so a TYPED NIL of this type is non-nil as an interface and panics +// when called. +type nilDerefStore struct { + storage.DoltStorage + cfg issueops.WorkspaceConfig +} + +func (s *nilDerefStore) WorkspaceConfig() (issueops.WorkspaceConfig, error) { + return s.cfg, nil +} + +// configUseCaseProbe is a unit of work's config use case with one canned answer. +type configUseCaseProbe struct { + domain.ConfigUseCase + value string + err error + reads int + keys []string +} + +func (c *configUseCaseProbe) GetConfig(_ context.Context, key string) (string, error) { + c.reads++ + c.keys = append(c.keys, key) + return c.value, c.err +} + +// unitProbe is a unit of work that records how it was finished: a settings read +// must be rolled back (Close), never committed. +type unitProbe struct { + uow.UnitOfWork + cfg *configUseCaseProbe + closed int + committed int +} + +func (u *unitProbe) ConfigUseCase() domain.ConfigUseCase { return u.cfg } +func (u *unitProbe) Close(context.Context) { u.closed++ } +func (u *unitProbe) Commit(context.Context, string) error { + u.committed++ + return nil +} + +// providerProbe is a unit-of-work provider that is a VersionedHistoryConfigurer. +type providerProbe struct { + uow.UnitOfWorkProvider + unit *unitProbe + newErr error + panicMsg string + newCalls int + enabled bool + sets int +} + +func (p *providerProbe) NewUOW(context.Context) (uow.UnitOfWork, error) { + p.newCalls++ + if p.panicMsg != "" { + panic(p.panicMsg) + } + if p.newErr != nil { + return nil, p.newErr + } + return p.unit, nil +} + +func (p *providerProbe) SetVersionedHistoryEnabled(enabled bool) { + p.enabled = enabled + p.sets++ +} + +func newProviderProbe(value string, err error) *providerProbe { + return &providerProbe{unit: &unitProbe{cfg: &configUseCaseProbe{value: value, err: err}}} +} + +// nonConfigurerProvider cannot record versions: it does not implement the capability. +type nonConfigurerProvider struct { + uow.UnitOfWorkProvider + newCalls int +} + +func (p *nonConfigurerProvider) NewUOW(context.Context) (uow.UnitOfWork, error) { + p.newCalls++ + panic("NewUOW on a provider that cannot record versions") +} + +// --- the resolver, moved from cmd/bd with its assertions unchanged ---------- + +// TestStoreSettingAnswersFalseForAStoreThatCannotAnswer pins the contract the +// function's own doc states, across every way a store can fail to produce a true +// -- including the two that are panics rather than errors, which is the pair that +// took the process down. +func TestStoreSettingAnswersFalseForAStoreThatCannotAnswer(t *testing.T) { + ctx := context.Background() + var typedNil *nilDerefStore + + for _, tt := range []struct { + name string + store storage.DoltStorage + want bool + }{ + {"nil store", nil, false}, + {"typed nil store", typedNil, false}, + {"WorkspaceConfig panics", &configProbeStore{panicMsg: "no settings plane on this store"}, false}, + {"WorkspaceConfig errors", &configProbeStore{err: fmt.Errorf("server unreachable")}, false}, + {"WorkspaceConfig returns no plane", &configProbeStore{}, false}, + {"GetSetting panics", &configProbeStore{cfg: &settingsPlane{panicMsg: "settings table missing"}}, false}, + {"GetSetting errors", &configProbeStore{cfg: &settingsPlane{err: fmt.Errorf("read failed")}}, false}, + {"unset is off", &configProbeStore{cfg: &settingsPlane{value: ""}}, false}, + {"stored false", &configProbeStore{cfg: &settingsPlane{value: "false"}}, false}, + {"unparseable is off", &configProbeStore{cfg: &settingsPlane{value: "yes please"}}, false}, + {"stored true", &configProbeStore{cfg: &settingsPlane{value: "true"}}, true}, + {"stored 1", &configProbeStore{cfg: &settingsPlane{value: "1"}}, true}, + {"stored true with whitespace", &configProbeStore{cfg: &settingsPlane{value: " true\n"}}, true}, + } { + t.Run(tt.name, func(t *testing.T) { + // No recover here on purpose: a panic escaping the call under test + // must fail this test, not be absorbed by it. + if got := StoreSetting(ctx, tt.store); got != tt.want { + t.Errorf("StoreSetting = %v, want %v", got, tt.want) + } + }) + } +} + +// TestParseSettingBool pins how the stored string is read back. `bd config set` +// stores whatever the user typed, verbatim, so this is the whole translation +// between that and the boolean the factories activate on. +func TestParseSettingBool(t *testing.T) { + for _, tt := range []struct { + in string + want bool + }{ + {"", false}, + {"true", true}, + {"TRUE", true}, + {"True", true}, + {"1", true}, + {"t", true}, + {" true ", true}, + {"false", false}, + {"0", false}, + {"off", false}, + {"on", false}, + {"nonsense", false}, + } { + t.Run(fmt.Sprintf("%q", tt.in), func(t *testing.T) { + if got := parseSettingBool(tt.in); got != tt.want { + t.Errorf("parseSettingBool(%q) = %v, want %v", tt.in, got, tt.want) + } + }) + } +} + +// TestEnabledForStoreSkipsTheStoreReadForANonConfigurer is the capability GUARD's +// test, and the regression test for the CI failure as it actually happened. +// +// The store read is only ever used to decide whether to call +// SetVersionedHistoryEnabled, so a store that cannot record versions has nothing to +// gain from being asked -- and, if its settings plane is absent or half-built, +// everything to lose. That is what took Test (macos-latest) and PR Core (wrapper +// timing) red at 32d4f966e, via cmd/bd's own fakeProbingDoltStore. +// +// The assertion that matters is calls == 0: not "it survived" (the recover would +// deliver that too, which is exactly how this regressed the first time) but "it was +// never asked". +func TestEnabledForStoreSkipsTheStoreReadForANonConfigurer(t *testing.T) { + raw := &nonConfigurerStore{} + if _, isConfigurer := storage.DoltStorage(raw).(storage.VersionedHistoryConfigurer); isConfigurer { + t.Fatal("nonConfigurerStore satisfies VersionedHistoryConfigurer; this test cannot exercise the guard it claims to") + } + if EnabledForStore(context.Background(), raw) { + t.Error("EnabledForStore = true for a store that cannot record versions") + } + if raw.calls != 0 { + t.Fatalf("WorkspaceConfig called %d times on a store that cannot record versions, want 0", raw.calls) + } +} + +// TestEnabledForStoreReadsTheStoreWhenItCanRecord is the other half: the guard must +// not be so eager that it stops reading the plane that finding 1 existed to start +// reading. A store that CAN record still gets asked, and a stored true still turns +// recording on. +func TestEnabledForStoreReadsTheStoreWhenItCanRecord(t *testing.T) { + raw := &configProbeStore{cfg: &settingsPlane{value: "true"}} + if got := EnabledForStore(context.Background(), raw); !got { + t.Fatal("EnabledForStore = false for a configurer whose store plane says true") + } + if raw.calls != 1 { + t.Fatalf("store read %d times, want 1", raw.calls) + } +} + +// TestEnabledReadsTheRowWithoutTheCapabilityGate: `bd versions` is the one caller +// where the read is the point, so it must ask a store that cannot record too. +func TestEnabledReadsTheRowWithoutTheCapabilityGate(t *testing.T) { + raw := &planeOnlyStore{cfg: &settingsPlane{value: "true"}} + if _, isConfigurer := storage.DoltStorage(raw).(storage.VersionedHistoryConfigurer); isConfigurer { + t.Fatal("planeOnlyStore satisfies VersionedHistoryConfigurer; this test cannot exercise what it claims to") + } + if !Enabled(context.Background(), raw) { + t.Error("Enabled = false for a store whose row says true") + } + if raw.calls != 1 { + t.Errorf("store read %d times, want 1", raw.calls) + } +} + +// --- each store answers for itself; the environment only turns recording on -- + +// TestActivateStoreReadsTheStoresOwnRow is D1 at the unit level: two stores opened +// by one process each read ITS OWN row. The one whose row says on records; the one +// with no row does not, and does not inherit the other's answer. +func TestActivateStoreReadsTheStoresOwnRow(t *testing.T) { + ctx := context.Background() + on := &configProbeStore{cfg: &settingsPlane{value: "true"}} + absent := &configProbeStore{cfg: &settingsPlane{value: ""}} + off := &configProbeStore{cfg: &settingsPlane{value: "false"}} + + for name, s := range map[string]*configProbeStore{"on": on, "absent": absent, "off": off} { + got, err := ActivateStore(ctx, s, nil) + if err != nil || got != storage.DoltStorage(s) { + t.Fatalf("%s: ActivateStore = (%v, %v), want the same store and no error", name, got, err) + } + } + if !on.enabled { + t.Error("the store whose row says true was not activated") + } + if absent.enabled || off.enabled { + t.Errorf("a store with no row (%v) or a false row (%v) was activated by another store's answer", absent.enabled, off.enabled) + } +} + +// TestActivateStoreReadsTheRowExactlyOncePerOpen: with the factories activating, +// wireStorageDecorators no longer does, so each store is activated and its row read +// once per open -- not twice. +func TestActivateStoreReadsTheRowExactlyOncePerOpen(t *testing.T) { + plane := &settingsPlane{value: "true"} + raw := &configProbeStore{cfg: plane} + if _, err := ActivateStore(context.Background(), raw, nil); err != nil { + t.Fatal(err) + } + if raw.calls != 1 || plane.reads != 1 { + t.Errorf("settings plane fetched %d time(s) and row read %d time(s), want 1 and 1", raw.calls, plane.reads) + } + if raw.sets != 1 || !raw.enabled { + t.Errorf("SetVersionedHistoryEnabled called %d time(s), enabled=%v; want exactly one call turning it on", raw.sets, raw.enabled) + } +} + +func TestActivateStorePassesAFailedOpenThrough(t *testing.T) { + openErr := errors.New("open failed") + + got, err := ActivateStore(context.Background(), nil, openErr) + if got != nil || !errors.Is(err, openErr) { + t.Errorf("ActivateStore(nil, err) = (%v, %v), want (nil, the open error)", got, err) + } + + // A store handed back alongside an error is not touched or read either. + raw := &configProbeStore{cfg: &settingsPlane{value: "true"}} + got, err = ActivateStore(context.Background(), raw, openErr) + if got != storage.DoltStorage(raw) || !errors.Is(err, openErr) { + t.Errorf("ActivateStore(store, err) = (%v, %v), want the store and the open error unchanged", got, err) + } + if raw.calls != 0 || raw.sets != 0 { + t.Errorf("a failed open still touched the store: %d settings reads, %d activations", raw.calls, raw.sets) + } +} + +// TestActivateStoreSurvivesAStoreThatCannotAnswer is the regression test for the CI +// failure itself, at the layer that now owns it. This is the RECOVER's test: the +// store here IS a configurer, so the capability guard lets it through to the read, +// and only the recover keeps the panic from escaping. +func TestActivateStoreSurvivesAStoreThatCannotAnswer(t *testing.T) { + raw := &configProbeStore{panicMsg: "WorkspaceConfig on a store with no settings plane"} + got, err := ActivateStore(context.Background(), raw, nil) + if err != nil || got != storage.DoltStorage(raw) { + t.Fatalf("ActivateStore = (%v, %v), want the store back and no error", got, err) + } + if raw.calls != 1 { + t.Fatalf("WorkspaceConfig called %d times, want 1; this test is not exercising the read it claims to", raw.calls) + } + if raw.enabled { + t.Error("a store whose settings plane panicked was switched on") + } +} + +func TestActivateStoreLeavesAStoreThatCannotRecordUntouched(t *testing.T) { + raw := &nonConfigurerStore{} + got, err := ActivateStore(context.Background(), raw, nil) + if err != nil || got != storage.DoltStorage(raw) { + t.Fatalf("ActivateStore = (%v, %v), want the store back and no error", got, err) + } + if raw.calls != 0 { + t.Errorf("a store that cannot record was asked for its settings plane %d time(s), want 0", raw.calls) + } +} + +// TestEnvironmentIsProcessWideAndShortCircuitsTheRead is D3 for stores: viper says +// on, so every store the process opens is activated -- including one whose own row +// says nothing -- and the row is not even read. +func TestEnvironmentIsProcessWideAndShortCircuitsTheRead(t *testing.T) { + viperEnv(t, "1") + + absent := &configProbeStore{cfg: &settingsPlane{value: ""}} + if _, err := ActivateStore(context.Background(), absent, nil); err != nil { + t.Fatal(err) + } + if !absent.enabled { + t.Error("the environment said on but a store with no row was not activated") + } + if absent.calls != 0 { + t.Errorf("the settings plane was read %d time(s) although the environment had already answered, want 0", absent.calls) + } +} + +// TestEnvironmentZeroNeverSwitchesOffAStoreWhoseRowIsOn is D3's other half, and +// the rule that OR is never ranked: =0 is not "off". A store whose own row says on +// keeps recording, and a store with no row stays quiet. The only way off is the row. +func TestEnvironmentZeroNeverSwitchesOffAStoreWhoseRowIsOn(t *testing.T) { + viperEnv(t, "0") + ctx := context.Background() + + on := &configProbeStore{cfg: &settingsPlane{value: "true"}} + absent := &configProbeStore{cfg: &settingsPlane{value: ""}} + for _, s := range []*configProbeStore{on, absent} { + if _, err := ActivateStore(ctx, s, nil); err != nil { + t.Fatal(err) + } + } + if !on.enabled { + t.Error("BD_VERSIONED_HISTORY_ENABLED=0 switched off a store whose own row says true") + } + if absent.enabled { + t.Error("BD_VERSIONED_HISTORY_ENABLED=0 switched ON a store with no row") + } +} + +// --- D7: an error is not an absent row -------------------------------------- + +func TestAnAbsentRowIsSilentlyOffButAReadErrorIsDiagnosed(t *testing.T) { + ctx := context.Background() + + t.Run("absent row: off, and nothing to say", func(t *testing.T) { + raw := &configProbeStore{cfg: &settingsPlane{value: ""}} + var got bool + out := captureStderr(t, true, func() { got = StoreSetting(ctx, raw) }) + if got { + t.Error("an absent row resolved to on") + } + if out != "" { + t.Errorf("an absent row is the normal state of every workspace and must not be diagnosed, got: %q", out) + } + }) + + t.Run("GetSetting error: off, and diagnosed at debug level", func(t *testing.T) { + raw := &configProbeStore{cfg: &settingsPlane{err: errors.New("read failed: table config is locked")}} + var got bool + out := captureStderr(t, true, func() { got = StoreSetting(ctx, raw) }) + if got { + t.Error("a failed row read resolved to on") + } + for _, want := range []string{ConfigKey, "table config is locked"} { + if !strings.Contains(out, want) { + t.Errorf("the diagnostic does not mention %q: %q", want, out) + } + } + }) + + t.Run("WorkspaceConfig error: off, and diagnosed at debug level", func(t *testing.T) { + raw := &configProbeStore{err: errors.New("server unreachable")} + var got bool + out := captureStderr(t, true, func() { got = StoreSetting(ctx, raw) }) + if got { + t.Error("an unreadable settings plane resolved to on") + } + if !strings.Contains(out, "server unreachable") { + t.Errorf("the diagnostic does not carry the cause: %q", out) + } + }) + + t.Run("the diagnostic is debug-level, never unconditional stderr", func(t *testing.T) { + raw := &configProbeStore{cfg: &settingsPlane{err: errors.New("read failed")}} + out := captureStderr(t, false, func() { _ = StoreSetting(ctx, raw) }) + if out != "" { + t.Errorf("this runs on every invocation and must be silent unless debugging, got: %q", out) + } + }) +} + +// --- Apply: the raw-store ordering, and the set-but-unsupported warning ----- + +// TestApplyReachesOnlyTheRawStore pins the ORDERING that activation depends on. +// +// The capability is reached by type assertion, and none of the decorators +// (telemetry, externaldeps, hooks) forwards SetVersionedHistoryEnabled. So applying +// the config after any wrap would assert against a wrapper, fail silently, and +// leave versioned history OFF while the operator's config said it was on -- a wrong +// answer rather than an empty one. That is why the factories, which hand back the +// concrete store before anything wraps it, are where activation lives. +// +// The second half of this test is the part that actually guards the ordering: it +// proves the wrapped chain does NOT satisfy the capability, so if a decorator ever +// starts forwarding, the constraint this test exists for has changed. +func TestApplyReachesOnlyTheRawStore(t *testing.T) { + raw := &configProbeStore{} + Apply(raw, true) + if !raw.enabled { + t.Error("Apply(raw, true) did not enable versioned history on the raw store") + } + + wrap := func(s storage.DoltStorage) storage.DoltStorage { + return externaldeps.New( + telemetry.WrapStorage(s), + func(externaldeps.ProjectName) (string, bool) { return "", false }, + func(context.Context, string) (storage.DoltStorage, error) { return nil, errors.New("unused") }, + ) + } + if _, ok := wrap(&configProbeStore{}).(storage.VersionedHistoryConfigurer); ok { + // Fatal, not Skip. A skip here returns BEFORE the assertion below -- the + // half that actually guards the ordering -- so the case would report + // "not applicable" while checking nothing. + t.Fatal("a decorator now forwards SetVersionedHistoryEnabled; the ordering constraint has changed and this test needs rewriting rather than silently passing") + } + + behindWrap := &configProbeStore{} + captureStderr(t, false, func() { Apply(wrap(behindWrap), true) }) + if behindWrap.enabled { + t.Error("the capability was somehow reached through the decorators; this test's premise is stale") + } +} + +// TestApplyOffIsANoop pins that the disabled path touches nothing at all -- +// flag-off must be byte-identical to a build without the feature, which is the +// contract #6135 ships under. +func TestApplyOffIsANoop(t *testing.T) { + raw := &configProbeStore{} + out := captureStderr(t, true, func() { Apply(raw, false) }) + if raw.sets != 0 || raw.enabled { + t.Errorf("Apply(raw, false) called SetVersionedHistoryEnabled %d time(s), enabled=%v; flag-off must be a no-op", raw.sets, raw.enabled) + } + if out != "" { + t.Errorf("Apply(raw, false) wrote %q", out) + } +} + +// TestApplyWarnsWhenTheBackendCannotRecord: asking for history and not getting it is +// exactly the case a user must be told about. The wording is #6661's, unchanged. +func TestApplyWarnsWhenTheBackendCannotRecord(t *testing.T) { + out := captureStderr(t, false, func() { Apply(&nonConfigurerStore{}, true) }) + for _, want := range []string{ + "warning: versioned-history.enabled is set, but this storage backend (", + "does not support version history; it stays off", + } { + if !strings.Contains(out, want) { + t.Errorf("the warning is missing %q: %q", want, out) + } + } +} + +// TestConfigKeyIsNotYamlOnly is D5's pin at the unit level. versioned-history.* must +// NOT become a yaml-only key: `bd config set versioned-history.enabled true` keeps +// writing the database row exactly as #6661 wired it, and a yaml-only key would move +// the switch off the store. +func TestConfigKeyIsNotYamlOnly(t *testing.T) { + if ConfigKey != "versioned-history.enabled" { + t.Errorf("ConfigKey = %q; the key is documented, and read back by `bd versions`, under this exact spelling", ConfigKey) + } + if config.IsYamlOnlyKey(ConfigKey) { + t.Errorf("%s is a yaml-only key, so `bd config set` would write config.yaml and no store would ever read it", ConfigKey) + } +} + +// --- K1: the provider path --------------------------------------------------- + +// TestActivateProviderReadsItsOwnRow is K1's central claim. A unit-of-work provider +// has no storage.DoltStorage to ask, so its flag comes from viper OR its own row -- +// read through a unit of work's ConfigUseCase -- and NOT from viper alone. +func TestActivateProviderReadsItsOwnRow(t *testing.T) { + p := newProviderProbe("true", nil) + got, err := ActivateProvider(context.Background(), p, nil) + if err != nil || got != uow.UnitOfWorkProvider(p) { + t.Fatalf("ActivateProvider = (%v, %v), want the provider back and no error", got, err) + } + if !p.enabled { + t.Fatal("a provider whose database row says true was not activated: the flag was resolved from viper alone") + } + if p.unit.cfg.reads != 1 || len(p.unit.cfg.keys) != 1 || p.unit.cfg.keys[0] != ConfigKey { + t.Errorf("config rows read = %v (%d read(s)), want exactly one read of %q", p.unit.cfg.keys, p.unit.cfg.reads, ConfigKey) + } + // A settings read is a read: rolled back, never committed. + if p.unit.closed != 1 || p.unit.committed != 0 { + t.Errorf("unit of work closed %d time(s) and committed %d time(s), want closed once and never committed", p.unit.closed, p.unit.committed) + } +} + +func TestActivateProviderLeavesAnAbsentOrFalseRowOff(t *testing.T) { + for _, value := range []string{"", "false", "0", "on", "nonsense"} { + t.Run(fmt.Sprintf("%q", value), func(t *testing.T) { + p := newProviderProbe(value, nil) + captureStderr(t, true, func() { _, _ = ActivateProvider(context.Background(), p, nil) }) + if p.enabled { + t.Errorf("a provider whose row is %q was activated", value) + } + if p.unit.closed != 1 { + t.Errorf("unit of work closed %d time(s), want 1", p.unit.closed) + } + }) + } + t.Run("an absent row is silent", func(t *testing.T) { + p := newProviderProbe("", nil) + if out := captureStderr(t, true, func() { _, _ = ActivateProvider(context.Background(), p, nil) }); out != "" { + t.Errorf("an absent row must not be diagnosed, got %q", out) + } + }) +} + +// TestActivateProviderReadFailureIsOffAndDiagnosed is D7 for providers: an error is +// not an absent row. It resolves to off -- but not silently. +func TestActivateProviderReadFailureIsOffAndDiagnosed(t *testing.T) { + t.Run("GetConfig errors", func(t *testing.T) { + p := newProviderProbe("", errors.New("no such table: config")) + out := captureStderr(t, true, func() { _, _ = ActivateProvider(context.Background(), p, nil) }) + if p.enabled { + t.Error("a failed row read activated the provider") + } + for _, want := range []string{ConfigKey, "no such table: config"} { + if !strings.Contains(out, want) { + t.Errorf("the diagnostic does not mention %q: %q", want, out) + } + } + if p.unit.closed != 1 { + t.Errorf("unit of work closed %d time(s) after a failed read, want 1", p.unit.closed) + } + }) + t.Run("NewUOW errors", func(t *testing.T) { + p := &providerProbe{newErr: errors.New("connection refused")} + out := captureStderr(t, true, func() { _, _ = ActivateProvider(context.Background(), p, nil) }) + if p.enabled { + t.Error("a provider that could not open a unit of work was activated") + } + if !strings.Contains(out, "connection refused") { + t.Errorf("the diagnostic does not carry the cause: %q", out) + } + }) + t.Run("silent unless debugging", func(t *testing.T) { + p := newProviderProbe("", errors.New("no such table: config")) + if out := captureStderr(t, false, func() { _, _ = ActivateProvider(context.Background(), p, nil) }); out != "" { + t.Errorf("this runs on every invocation and must be silent unless debugging, got: %q", out) + } + }) +} + +func TestActivateProviderSurvivesAProviderThatPanics(t *testing.T) { + p := &providerProbe{panicMsg: "NewUOW on a half-built provider"} + got, err := ActivateProvider(context.Background(), p, nil) + if err != nil || got != uow.UnitOfWorkProvider(p) { + t.Fatalf("ActivateProvider = (%v, %v), want the provider back and no error", got, err) + } + if p.newCalls != 1 { + t.Fatalf("NewUOW called %d times, want 1; this test is not exercising the read it claims to", p.newCalls) + } + if p.enabled { + t.Error("a provider whose read panicked was activated") + } +} + +func TestActivateProviderSkipsTheReadForANonConfigurer(t *testing.T) { + p := &nonConfigurerProvider{} + if _, ok := uow.UnitOfWorkProvider(p).(storage.VersionedHistoryConfigurer); ok { + t.Fatal("nonConfigurerProvider satisfies VersionedHistoryConfigurer; this test cannot exercise the guard it claims to") + } + got, err := ActivateProvider(context.Background(), p, nil) + if err != nil || got != uow.UnitOfWorkProvider(p) { + t.Fatalf("ActivateProvider = (%v, %v), want the provider back and no error", got, err) + } + if p.newCalls != 0 { + t.Errorf("a provider that cannot record was asked for a unit of work %d time(s), want 0", p.newCalls) + } +} + +func TestActivateProviderPassesAFailedOpenThrough(t *testing.T) { + openErr := errors.New("open failed") + got, err := ActivateProvider(context.Background(), nil, openErr) + if got != nil || !errors.Is(err, openErr) { + t.Errorf("ActivateProvider(nil, err) = (%v, %v), want (nil, the open error)", got, err) + } + p := newProviderProbe("true", nil) + got, err = ActivateProvider(context.Background(), p, openErr) + if got != uow.UnitOfWorkProvider(p) || !errors.Is(err, openErr) { + t.Errorf("ActivateProvider(provider, err) = (%v, %v), want the provider and the open error unchanged", got, err) + } + if p.newCalls != 0 || p.sets != 0 { + t.Errorf("a failed open still touched the provider: %d unit(s) of work opened, %d activation(s)", p.newCalls, p.sets) + } +} + +// TestProviderEnvironmentIsProcessWideAndOnlyTurnsRecordingOn is D3 for providers. +func TestProviderEnvironmentIsProcessWideAndOnlyTurnsRecordingOn(t *testing.T) { + t.Run("=1 activates without reading the row", func(t *testing.T) { + viperEnv(t, "1") + p := newProviderProbe("", nil) + if _, err := ActivateProvider(context.Background(), p, nil); err != nil { + t.Fatal(err) + } + if !p.enabled { + t.Error("the environment said on but the provider was not activated") + } + if p.newCalls != 0 { + t.Errorf("a unit of work was opened %d time(s) although the environment had already answered, want 0", p.newCalls) + } + }) + t.Run("=0 never switches off a provider whose row is on", func(t *testing.T) { + viperEnv(t, "0") + p := newProviderProbe("true", nil) + if _, err := ActivateProvider(context.Background(), p, nil); err != nil { + t.Fatal(err) + } + if !p.enabled { + t.Error("BD_VERSIONED_HISTORY_ENABLED=0 switched off a provider whose own row says true") + } + }) +} diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index b7e0b6570..410728ae0 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -290,6 +290,7 @@ release_cross_build( "//internal/utils:utils", "//internal/validation:validation", "//internal/versioncheck:versioncheck", + "//internal/versionedhistory:versionedhistory", "//internal/workapi/storecounter:storecounter", "//internal/workapi/storequerier:storequerier", "//internal/workapi/storereader:storereader", diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index 4b685968a..db584cf99 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -69,6 +69,7 @@ import sys # Packages whose own Go source a test scans. PACKAGES = ( + ".", # //cmd/bd:bd_test (events-journal construction scan: the Go SDK opens) "backend", # //backend:backend_test (public alias census) "backend/conformance", # //backend/conformance, //internal/storage "beadserrors", # role facade alias targets From 5c9c02ab77a3e52a861e45f0b9712628eba8ad34 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sun, 18 Oct 2026 21:20:57 +0000 Subject: [PATCH 15/54] Phase 3 (R20): epoch-transition enforcement + minted-address CAS - gastownhall/beads#6136 (#6664) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(conformance): un-skip R16 CAS + R20 epoch tests with honest not-implemented stubs Wires all 10 ExpectedRevisionFixture-gated Run* functions and both EpochFixture-gated Run* functions to real (non-nil) hooks on all three storage legs (dolt, embeddeddolt, uow), replacing the Phase 0 all-nil fixtures that made these cases t.Skip. Every wired hook is an unconditional "not implemented yet" stub rather than a partial-real implementation. This is deliberate: every Run* function under test calls its primary hook (CompareAndSetVersion, or CurrentEpoch/MintUnderEpoch) first, before any other hook, so a stub that errors unconditionally on that first call guarantees deterministic RED with zero risk of an accidental pass -- a concern that's concrete, not hypothetical, since at least one case (RunExpectedRevisionAcceptsAWriteNamingNoVersion) would go green under a partially-real read/unconditional-write strategy backed by Phase 1's already-persisted current_revision/store_epoch columns. Each case now fails for its documented reason: most via a direct t.Fatalf quoting the stub's "not implemented" error, and two (RefusalIsDistinguishableFromNotFound, RefusalIsDistinguishableFromValidationFailure) via a clean want/got mismatch against the sentinel errors they expect. Explicitly out of scope and untouched: CrossRecordInvariantFixture (R16.1) and the non-epoch RetentionFixture cases (R17) -- confirmed via test run that TestRetentionContract still reports PASS with all 13 subtests SKIP on every leg. TestEveryLegWiresEveryRoleContract (internal/storage/leg_contract_wiring_test.go) is a source-level AST naming check independent of hook nil-ness; all 12 entrypoints were already called by every leg pre-change, so this lock was satisfied before and remains satisfied after. Sentinel error messages reference be-x5jqd.3 (R16 CAS implementation) and be-x5jqd.4 (R20 epoch enforcement implementation), the two builder children this un-skip unblocks. Verified via go build ./..., go vet ./..., and go test -tags cgo across all three legs: all 10 R16 + 2 R20 subtests FAIL for the documented reason, TestRetentionContract unaffected. Bead: be-x5jqd.1 * feat: green — Implement R20 epoch-transition enforcement (refs be-x5jqd.4) * fix(epoch): honor R20-n's retained-mapping exception in StillServes/Resolve StillServesInTx and ResolveEpochInTx only ever checked whether the literal queried address's own row matched the current epoch. Per R20-n (#5898 rev 9), a version that survives an epoch transition keeps its prior-epoch address resolving once its id has been carried forward into the current epoch (via CurrentAddressForInTx or a fresh MintUnderEpochInTx) — that carried-forward row wasn't being consulted, so a legitimately retained address would wrongly stop resolving. Add mintedIDHasAddressAtEpochInTx and consult it as a fallback in both functions when the queried address's own row is stale. Addresses whose id was never carried forward are unaffected (row.mintedID has no row at the current epoch, same false/GoneReorganization answer as before). Round-2 response to be-rskwc's causal_disproof finding: the merge-base commit FAILed honestly on this defect, this branch's tdd_green SKIPped instead (attribution_evidence). This fix is real and verified via a new sqlmock unit test (TestStillServesInTxHonorsARetainedMapping et al.), but does not by itself flip TestEpochContract/EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed from SKIP to PASS: that test's own construction mints two records identically and never carries either forward before comparing StillServes on both, so no backend implementation can make them differ (escalation follows on a needs-mayor bead). refs be-x5jqd.4, be-rskwc round 2 * test(feat): red — harden EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed (refs be-9fiaw) Per mayor ruling on be-9fiaw: fix the test rather than waive it. The fixture already lets record-a survive an epoch bump by minting it again under the new epoch; record-b, never re-minted, does not survive. That asymmetry was previously only discovered after the fact (skip if StillServes agreed for both addresses), which meant the case could pass by observing either polarity. Assert the concrete expected values instead: servesA must be true, servesB must be false. This makes the case deterministically exercise both halves of R20-n, and closes a latent gap where a backend with the polarity reversed could have silently passed via the old discover-then-swap logic. * fix(schema): renumber epoch_minted_addresses to 0070, linearizing our own slot collision (refs be-hs42e.7) R20's epoch_minted_addresses and R7.1's removed_restriction both claimed migration slot 0069 on parallel sibling branches. Neither contained the other, so nothing in either branch's own CI could see the union: each was green alone, and the second to merge would panic checkNoDuplicateVersions at store open -- before any command's RunE, so every bd invocation bricks rather than one package reddening. Same hazard bee reproduced for #6358 vs #6008. This linearizes them instead of leaving it to merge order. R7.1 keeps 0069: it is published beneath #6661 and cannot move without rewriting a branch a reviewer is reading. R20 stops being a sibling, rebases onto R7.1, and takes the next contiguous slot. Contiguity is why the renumber could not be done pre-emptively in either branch alone: TestAllMigrationsSQLAppliesThroughDoltCLIAndRecordsLatestVersion asserts COUNT(*) == LatestVersion(), so a jump to a far slot fails until the gap is filled. The order had to be settled first. - migrations/0069_add_epoch_minted_addresses.{up,down}.sql -> 0070_* - migration_0069_add_epoch_minted_addresses_test.go -> migration_0070_*, with TestLatestVersionIncludesMigration0069 -> ...0070 (want 70) - 0069_add_removed_restriction_test.go retires its own LatestVersion pin in favour of 0070's, keeping the one-pin-at-a-time chain 0068 -> 0069 -> 0070 cli_migrations.go needs no change: its only override is R7.1's guarded PREPARE, which keeps 0069. The epoch migration is a plain unguarded CREATE TABLE IF NOT EXISTS and needs none. Verified: TestEmbeddedMigrationSourcesHaveNoDuplicateVersions, TestLatestVersionIncludesMigration0070, TestAllMigrationsSQLAppliesThroughDoltCLIAndRecordsLatestVersion (applies the full 0001..0070 chain through the dolt CLI), TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified and TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities all pass. go build ./... and go vet ./... clean. Co-Authored-By: Claude Opus 5 (1M context) * test(feat): red — Fix PR #6664: restore R16 honest-skip (stale be-x5jqd.3 premise) + address bee's 2 blockers on epoch_cas.go (refs be-ah0jf) * feat: green — Fix PR #6664: restore R16 honest-skip (stale be-x5jqd.3 premise) + address bee's 2 blockers on epoch_cas.go (refs be-ah0jf) * test(feat): red — Review: Fix PR #6664: restore R16 honest-skip (stale be-x5jqd.3 premise) + address bee's 2 blockers on epoch_cas.go (refs be-yj55d) * feat: green — Review: Fix PR #6664: restore R16 honest-skip (stale be-x5jqd.3 premise) + address bee's 2 blockers on epoch_cas.go (refs be-yj55d) * docs(epoch): cite epoch_minted_addresses as migration 0071 Upstream's 0069_widen_issue_versions_datetime_precision (#6675) took slot 0069 first, so removed_restriction moved to 0070 and epoch_minted_addresses to 0071. The three epoch_cas.go headers still named 0070. Rebase note (maintainer): the migration is now 0072, per the landing-order ruling (#6943's participation_generation is 0070 and #6661's removed_restriction is 0071); the commit "chore(schema): renumber epoch_minted_addresses to 0072 after #6661's 0071" updates these three headers. Co-Authored-By: Claude Opus 5.5 (1M context) * test(feat): red — move #6664's Dolt server leg to post-transaction publish for store_epoch and epoch_minted_addresses (latent lost-update pattern) Tests only. On the commit before this one they fail for these reasons: - the server leg publishes store_epoch / epoch_minted_addresses inside the open SQL transaction (DOLT_ADD before COMMIT) for BumpEpoch, MintUnderEpoch and CurrentAddressFor: the lost-update hazard documented on doltAddAndCommitInTx - MintUnderEpoch and CurrentAddressFor stage only epoch_minted_addresses, so the first mint leaves store_epoch modified in the working set, on both legs (#6664 Major 2, reproduced with Status() on a real server and embedded) - BumpEpoch must surface a Dolt commit that fails after its SQL commit, while the mint paths keep their rows - ensureStoreEpochRow seeds with INSERT IGNORE and reads the row back TestMintOverlappingABumpCommitsAnAddressThatIsAlreadyGone characterizes current behaviour and passes now: a mint overlapping a bump commits an address the bump already voided. Its doc comment records why that is left as it is. * feat: green — move #6664's Dolt server leg to post-transaction publish for store_epoch and epoch_minted_addresses (latent lost-update pattern) Server leg (internal/storage/dolt/epoch_cas.go): BumpEpoch, MintUnderEpoch and CurrentAddressFor no longer stage and Dolt-commit inside the open SQL transaction. The two mint paths publish through runIssueOperationTxWithMessage, which commits the SQL transaction first. BumpEpoch commits it, then publishes with doltAddAndCommitPostTx and RETURNS a failed Dolt commit: a bump that is committed but unpublished must not read as success. Both legs: a mint stages store_epoch with epoch_minted_addresses, because the first mint inserts the singleton row (#6664 Major 2). The set lives in issueops.EpochMintDirtyTables so the legs cannot drift from what the body writes. issueops: ensureStoreEpochRow seeds with INSERT IGNORE and reads the row back, as version_history.go does for the same row. Not changed, on purpose: a mint overlapping a concurrent bump can still commit an address the bump already voided. Dolt has no lock for it and the only conflicting write would overwrite bumped_at/bumped_reason; see the doc comment on TestMintOverlappingABumpCommitsAnAddressThatIsAlreadyGone. One comment in that test file is reworded; no test logic changed since the red commit. * chore(bazel): bazel-sync (gazelle) for the two new epoch test files Only the two go_test srcs entries for the test files this branch adds: epoch_cas_publish_test.go (dolt) and epoch_cas_status_test.go (embeddeddolt). gazelle wanted nothing else for them (no deps change). * test(conformance): add the optional EpochFixture.LoseVersion capability Record survival explicitly instead of inferring it (gastownhall/beads#6664, review 5360880888, Major 1). Once an address stays served unless its own row says the store lost it, the only way for the contract to produce a Gone address is an explicit loss, so the contract needs a way to say "the store lost this Version". LoseVersion is additive and optional: no existing field changes signature, and a nil LoseVersion makes the cases that need one skip with a named reason. It is its own commit so that it can be judged, or replaced, on its own: the same semantics could ride on BumpEpoch taking a lost set, and only the plumbing would differ. Nothing uses it yet. * chore(epoch): add signature-only stubs for the survival rework The tests that follow exercise BumpEpochCarryingInTx, LoseVersionInTx, ErrEpochAddressNotServed and the BumpEpochCarrying and LoseVersion store methods (gastownhall/beads#6664, review 5360880888). Stubs give those names so the tree builds and the tests fail for their real, behavioural reasons instead of failing to compile. No logic: ErrEpochAddressNotServed is a sentinel, BumpEpochCarrying advances the counter exactly as BumpEpoch does and carries nothing, and LoseVersion returns "not implemented". The next commit adds the tests; the one after replaces these bodies. * test(feat): red — record survival explicitly in epoch_minted_addresses, make CurrentAddressFor read-only, carry on a token-scheme change Tests for the survival rework of gastownhall/beads#6664 (review 5360880888: Major 1, Minors 3 and 4). Every test is written before any production change. - Contract: the earlier mint-driven R20-n case is restated as three named phases (a survivor and a lost Version, loss is terminal, survival needs no upkeep), plus three new cases for a token-scheme change (carry the live lineages and only those, a carried address keeps resolving across later bumps, a lost address stays Gone across a scheme change). All three legs register them and wire LoseVersion and the carrying bump. - Shared body: unit tests for the row-based resolver (exactly the address's row and the epoch, fail closed on a row from a future epoch), a read-only CurrentAddressFor with typed errors, LoseVersion, a bump that writes only store_epoch, a mint that never hands back a lost address, and the per-database counter (Minor 3). Minor 4 gets a guard that no address in the test file is written by hand. - Real engine: the carry on the embedded leg (one carry row per live id, the lineage root, the collision rule, a row from a future epoch, a rewound counter, database-wide, lost lineages), the publication sets of every writer, and on the server leg the overlap of a mint with a bump (now a Live survivor) and a deterministic mint-after-loss interleaving that must not revive the lost address. - Migration 0071 byte pin and a real-dolt shape check for carried_from, gone_at_epoch, the carried_from index and the gone-not-before-mint CHECK. These fail on the current code, and some do not build yet because BumpEpochCarryingInTx, LoseVersionInTx, ErrEpochAddressNotServed and the store methods they exercise do not exist. * feat: green — record survival explicitly in epoch_minted_addresses, make CurrentAddressFor read-only, carry on a token-scheme change The survival rework of gastownhall/beads#6664 (review 5360880888: Major 1, Minors 3 and 4). An address's status is now a function of its own row and the current epoch, never inferred from other rows. - Migration 0071 (edited in place; no new slot, no ALTER): the table gains carried_from, gone_at_epoch, an index on carried_from and a CHECK that a loss epoch is not before the mint epoch. - Shared body: Live unless gone_at_epoch is set, and a row from an epoch that has not happened yet fails closed; the chain query is deleted. CurrentAddressFor only reads and returns typed errors. LoseVersion marks a whole lineage lost, once, guarded by gone_at_epoch IS NULL. BumpEpochCarrying (token-scheme change) writes one carry row per live lineage naming the lineage root, resolves a shared-id collision by carrying the newest and losing the rest, and fails if a row already sits at the new epoch. A mint only inserts its own root row and never hands back an address that is not served. A restore or reinit bump still writes only store_epoch. - Both Dolt legs: BumpEpochCarrying publishes store_epoch and epoch_minted_addresses, LoseVersion publishes epoch_minted_addresses and returns a failed publication, and CurrentAddressFor runs on the read path and publishes nothing. The per-database epoch counter (Minor 3) and the requirement that a caller of the carrying bump hold the workspace gate exclusively are documented where they apply. One change to tests written in the previous commit: the three real-server publication probes that only need a clean working set now open a database of their own, as the overlap tests do, instead of queueing on the shared test database's schema-init lock. Its five-second acquire timeout ran out under load and failed two of them; no assertion changed. * test(dolt): run the epoch contract on a database of its own TestEpochContract set up through setupTestStore, which puts every parallel test on the shared test database and queues their schema initialization on one lock with a five-second acquire timeout. Under host load a setup that waits behind another parallel test's runs out of it and fails at its first line with "schema migration lock unavailable: timeout". That failed the unchanged first-mint test twice in the dolt lane once this change had added tests to it, and the contract's own setup is exposed to the same timeout. The contract now opens a database of its own, the way the overlap tests and the publication probes already do, so this change adds no contention on the shared database's lock. No assertion changed. * docs: replace internal tracker ids in the epoch comments and test names with plain descriptions Comment and text edits only; no behavior changes and no line is added or removed (21 lines are replaced one for one). Each replaced line named an internal tracker id in a header, a test comment, a failure message or the migration header; it now says what it meant in plain words or cites the upstream issue or pull request. The EpochFixture API lines and the already published commit messages are left as they were. * ci: list this PR's embedded storage tests in the embedded shard manifest #7156 made the embedded-Dolt tier's shard manifests exact-once (50 cmd shards, 15 storage shards), so every embedded test must be listed. This PR adds eleven embedded storage tests and no embedded cmd tests; this commit is exactly what the generators write for them: python3 scripts/ci/gen_embedded_cmd_shard_manifest.py 50 --weights=duration --write python3 scripts/ci/gen_embedded_storage_shard_manifest.py 15 --weights=duration --write Incremental: 11 lines added, none moved; the cmd manifest is unchanged. Rebase note (maintainer): regenerated on current main, whose embedded storage manifest uses a 40-shard block; the generator's incremental output placed these tests at shards 7, 9, 11, 20, 21, 28, 31, 32, 33, 34 and 36 of 40, one test per shard (gen_embedded_storage_shard_manifest.py 40 --weights=duration --write; --check passes). * chore(bazel): bazel-sync for the epoch sources and migration 0071 The BUILD files did not list epoch_cas.go in the dolt, embeddeddolt and issueops libraries, issueops' epoch_cas_test.go, or migration 0071 and its test, so Bazel could not compile those packages' tests. This is exactly the patch make bazel-sync writes (gazelle plus go_srcs.py; the PR's BUILD-sync CI step printed it): 4 files, 6 lines, nothing else. Rebase note (maintainer): the migration is now 0072, per the landing-order ruling (#6943's participation_generation is 0070 and #6661's removed_restriction is 0071); the commit "chore(schema): renumber epoch_minted_addresses to 0072 after #6661's 0071" renames these BUILD entries. On the rebased base, this commit's internal/storage/schema/BUILD.bazel hunk is merged with the base's 0070 and 0071 entries. * chore(schema): renumber epoch_minted_addresses to 0072 after #6661's 0071 Migrations are numbered by landing order. #6943 (participation_generation, the versioned-history write fence) owns main-plane 0070 and #6661 (removed_restriction) owns 0071, so this PR's epoch_minted_addresses migration moves to the next free slot, 0072. The rename is done here, as one maintainer commit, rather than inside the replayed commits, so the history shows exactly what changed relative to the PR as reviewed. - migrations/0071_add_epoch_minted_addresses.{up,down}.sql become 0072_*. Neither header names its own number, so the SQL and its comments are unchanged. Neither file exists on main, #6943 or #6661, so this is not an edit to a shipped migration. - cli_migrations.go needs nothing: the migration is a plain CREATE TABLE IF NOT EXISTS with no CLI-bundle override. - migration_0071_add_epoch_minted_addresses_test.go becomes migration_0072_add_epoch_minted_addresses_test.go; its consts, tests and messages say 0072, it owns the one LatestVersion pin (72), and its slot history now records the move past #6943's 0070. TestLatestVersionIncludesMigration0071 in #6661's removed_restriction test was already retired to a pointer comment by the rebase's resolution of "renumber epoch_minted_addresses to 0070"; that comment now names TestLatestVersionIncludesMigration0072 and its file. - BUILD.bazel embedsrcs/srcs, and the three epoch_cas.go headers (dolt, embeddeddolt, issueops) that cite the migration number. The intermediate commits of the rebase are not expected to build, for two reasons: - a duplicate migration version: the PR's epoch_minted_addresses migration shares 0069 with main's 0069_widen_issue_versions_datetime_precision until the PR's own "renumber epoch_minted_addresses to 0070", and from there shares 0071 with #6661's removed_restriction until this commit; - a helper-name collision: from the PR's "feat: green — Fix PR #6664: restore R16 honest-skip ... (refs be-ah0jf)" on, internal/storage/issueops declares readStoreEpochInTx twice, once in #6943's version_history.go and once in this PR's epoch_cas.go, until "fix(epoch): rename #6664's read-only store_epoch read past #6943's readStoreEpochInTx" later in this stack. This commit ends the duplicate version, but it still does not compile on its own because of the collision; the stack builds from the rename commit on. Checked at the tip of the stack, where the rename has landed: go test ./internal/storage/schema/... with the dolt CLI lane (including TestAllMigrationsSQLAppliesThroughDoltCLIAndRecordsLatestVersion and TestMigration0072RecordsSurvivalInTheAddressTableThroughDoltCLI), and scripts/check-migration-hygiene.sh with BASE_SHA=origin/main and BASE_SHA=vh/6661-rebase. Co-Authored-By: Claude Opus 5.5 (1M context) * test(epoch): waive #6664's three new epoch contracts on main's http leg main gained an HTTP storage leg (#7247) after this PR's base, and TestEveryLegWiresEveryRoleContract requires every leg to name or waive every role contract. This PR adds three EpochFixture cases (RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose, RunACarriedAddressKeepsResolvingAcrossLaterBumps and RunALostAddressStaysGoneAcrossASchemeChange); dolt, embeddeddolt and uow wire them, and the http leg did not name them. They are waived on the http leg with the existing retentionEpochNoRoleWaiverReason, which already covers every other RetentionFixture/EpochFixture case there: EpochFixture is raw storage hooks (CurrentEpoch, BumpEpoch, MintUnderEpoch, StillServes, Resolve, CurrentAddressFor, LoseVersion) with no publicops role behind them, and internal/httpclient is a remote client with no epoch counter or address table of its own. The constant's doc comment now counts eighteen cases. Test-only; no production code changes. After this, the http leg's only unnamed entrypoints are #6943's seven RunParticipationGeneration* contracts, whose waivers belong to #6943. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(epoch): rename #6664's read-only store_epoch read past #6943's readStoreEpochInTx #6943's "a create stamps participation even when the row is flagged no-history" (7c00d65fa6) adds readStoreEpochInTx to version_history.go: the mint's read of store_epoch, which SEEDS the singleton row when it is absent. This PR's epoch_cas.go already declared a readStoreEpochInTx of its own, so the package stopped compiling once #6664 was stacked on that commit. The two cannot be folded into one. This PR's helper is deliberately read-only: an absent row answers epoch 1 without writing, so CurrentEpochInTx, StillServesInTx, ResolveEpochInTx, CurrentAddressForInTx and LoseVersionInTx never write from a read (review item B4, pinned by TestCurrentAddressForNeverReseedsAnAbsentStoreEpochRow and TestCurrentAddressForWritesAndPublishesNothing). Calling #6943's seeding read from those paths would break that. So #6664's helper becomes readStoreEpochWithoutSeedingInTx, and its doc comment says why it is not readStoreEpochInTx. The five call sites and the ensureStoreEpochRow comment that names it follow. No behavior changes on either side: #6943's mint path keeps its seeding read, and this PR's read-only paths keep theirs. (ensureStoreEpochRow, which BumpEpochInTx and MintUnderEpochInTx use, now has the same body as #6943's readStoreEpochInTx; it is left as it is rather than folded in here.) Co-Authored-By: Claude Opus 5.5 (1M context) * test(storage): say exactly why the retention and epoch contracts stay waived on http All but four of the participation-generation contracts (#6943's write fence) now run over the served wire (internal/httpclient/served_participation_generation_test.go): their subject is a create or an update, which the client can send, and the fixture reads the reference store's raw columns. The four stay waived, because the mutation each needs has no form on the wire. This checked whether the eighteen RetentionFixture/EpochFixture cases waived on the http leg, including the three this PR adds, can be wired the same way. They cannot, cheaply or otherwise, without a new wire operation: - every EpochFixture hook (CurrentEpoch, BumpEpoch, MintUnderEpoch, StillServes, Resolve, CurrentAddressFor, LoseVersion) is the operation under test, not a seed or a raw read. Those operations are methods on the concrete *DoltStore and *EmbeddedDoltStore over issueops' *InTx bodies, on no storage interface, no publicops role and no wire operation. Every call path into them starts in a test: the methods are called only from each store leg's own _test.go files (its EpochFixture, which the conformance contracts drive, and its epoch tests), and the *InTx bodies only from those methods, from each other and from tests (the uow leg's fixture through a test-only type assertion, and issueops' and the dolt leg's own tests). No command, role or server handler reaches them, so no client mutation touches epoch state either; - RetentionFixture's hooks (Resolve, Remove, Hold, ForceRemove, Erase, Mint) are implemented by no backend yet (every store leg wires them nil and its cases skip), and the wire has no removal, hold, erasure or gone/unknown resolve to bind them to. Binding the epoch hooks to the reference store behind the served env would pass, but as a property of that store, which the store legs already pin, never of the client. So the waivers stay; retentionEpochNoRoleWaiverReason's comment and reason string now say the above, in the shape #6661 gave asOfReadNoHTTPWireWaiverReason, instead of the earlier general "raw storage hooks with no publicops role" wording. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(epoch): report applied-but-unpublished epoch writes as indeterminate, bound address length, verify dup-key mints Maintainer fixup for the #6664 review (adopt-pr attempt 1, iteration 1). Major 1, applied writes reported as failed. An epoch writer whose SQL commit has landed, or may have, now wraps storage.ErrCommitIndeterminate on both legs and hands back what it applied, so nothing replays a bump that has already advanced the counter: - embedded BumpEpoch and BumpEpochCarrying return the new epoch with the error (failedBump), and MintUnderEpoch its address; - server bumps wrap the sentinel on every Dolt failure after the SQL commit, including a failed staged-set read (unpublishedBump), and so does LoseVersion; an indeterminate SQL commit returns the epoch or address it may have committed, and a definite (typed MySQL) commit error returns neither. Tests: TestEpochWritesReportAnUnpublishedCommitAsIndeterminate (new, embedded, through an afterSQLCommit seam in runTransactionWithMessage); on the server leg TestEpochWritesMarkAFailedStagedReadIndeterminate, TestEpochWritesReturnWhatAnIndeterminateSQLCommitMayHaveApplied and TestEpochWritesReturnNothingWhenTheSQLCommitFailsDefinitely, and the sentinel asserted in every publication-failure test. Major 2, address overflow. MintUnderEpochInTx refuses a pair whose address would outgrow epoch_minted_addresses.address (VARCHAR(255)) at the widest epoch the counter can reach, with a typed ErrEpochAddressTooLong, so a carry can never fail on a lineage that gained an epoch digit. Major 3, duplicate-key mint. A duplicate key from the mint's INSERT is no longer success: the row is read again and the mint stands only if this transaction sees it served; a Gone row fails the mint with ErrEpochAddressNotServed, an unreadable one with the duplicate-key error. Measured on dolt-sql-server 2.2.0, the stale-snapshot INSERT does not report a duplicate key at all: its COMMIT fails with Error 1213 and the retried mint answers ErrEpochAddressNotServed (TestAMintRacingTheMintAndLossOfItsAddressConflictsAtCommit). Minors: - absent store_epoch row: sqlmock cases pin that CurrentEpoch, StillServes, Resolve, CurrentAddressFor and LoseVersion never seed it; switching any of the five to the seeding reader now fails the test; - the server publication tests mint their address instead of writing it by hand, and TestEpochCASTestsBuildEveryAddressWithEpochAddress scans that file too (issueops_test gains //internal/storage/dolt:go_srcs as data); - stale text: the duplicate-key and carry comments, the issueops header's CompareAndSetVersionInTx citation, the embedded header's parity claim, and the 0072 header's "upserts the same row" (0072 is past the frozen floor). Declined, for the owner: - a scheme-change carry voids an older live lineage re-minted after a restore: rule (a), document and pin it, or rule (b), resolve the loser to the winner's carry, is a design call about R20-n, not a review fix; - the uow epoch fixtures' in-transaction DOLT_COMMIT('-Am'): test-only fixtures with no production caller; routing them through RunTxScoped is follow-up work. Validation: go build ./...; go vet; nogo (bazel --config=nogo) on issueops, dolt and embeddeddolt; issueops epoch tests; the dolt epoch tests against a real dolt-sql-server (56 pass) and the fake-backed publication tests; embedded epoch tests with BEADS_TEST_EMBEDDED_DOLT=1; uow epoch contract; the schema package; the address guard under bazel in the local sandbox. All 14 overlay mutants of the fixes are killed. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Jim Wordelman Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Julian Knutsen Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- .../scripts/embedded-storage-test-shards.txt | 12 + .../conformance/retention_epoch_contract.go | 453 +++++-- .../versioned_history_wiring_test.go | 9 + internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/epoch_cas.go | 248 ++++ .../storage/dolt/epoch_cas_publish_test.go | 949 ++++++++++++++ .../dolt/expected_revision_contract_test.go | 20 +- .../dolt/retention_epoch_contract_test.go | 141 +- internal/storage/embeddeddolt/BUILD.bazel | 3 + internal/storage/embeddeddolt/epoch_cas.go | 197 +++ .../epoch_cas_publish_internal_test.go | 141 ++ .../embeddeddolt/epoch_cas_status_test.go | 562 ++++++++ .../expected_revision_contract_test.go | 20 +- .../retention_epoch_contract_test.go | 125 +- internal/storage/embeddeddolt/transaction.go | 6 + internal/storage/issueops/BUILD.bazel | 3 + internal/storage/issueops/epoch_cas.go | 680 ++++++++++ internal/storage/issueops/epoch_cas_test.go | 1141 +++++++++++++++++ internal/storage/leg_contract_wiring_test.go | 63 +- internal/storage/schema/BUILD.bazel | 2 + ...ation_0071_add_removed_restriction_test.go | 18 +- ...on_0072_add_epoch_minted_addresses_test.go | 184 +++ .../0072_add_epoch_minted_addresses.down.sql | 1 + .../0072_add_epoch_minted_addresses.up.sql | 64 + .../uow/expected_revision_contract_test.go | 20 +- .../uow/retention_epoch_contract_test.go | 205 ++- 26 files changed, 5135 insertions(+), 134 deletions(-) create mode 100644 internal/storage/dolt/epoch_cas.go create mode 100644 internal/storage/dolt/epoch_cas_publish_test.go create mode 100644 internal/storage/embeddeddolt/epoch_cas.go create mode 100644 internal/storage/embeddeddolt/epoch_cas_publish_internal_test.go create mode 100644 internal/storage/embeddeddolt/epoch_cas_status_test.go create mode 100644 internal/storage/issueops/epoch_cas.go create mode 100644 internal/storage/issueops/epoch_cas_test.go create mode 100644 internal/storage/schema/migration_0072_add_epoch_minted_addresses_test.go create mode 100644 internal/storage/schema/migrations/0072_add_epoch_minted_addresses.down.sql create mode 100644 internal/storage/schema/migrations/0072_add_epoch_minted_addresses.up.sql diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index d127ad9a6..cef22090a 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -205,6 +205,7 @@ 40 7 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates 40 7 TestEmbeddedLenientOpenWarning_DataBehind 40 7 TestEmbeddedRelationsLeavesTheCallersRequestAlone +40 7 TestLoseVersionLeavesTheEpochTablesCommitted 40 7 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict 40 8 TestDeleterContract @@ -217,6 +218,7 @@ 40 8 TestGetIssueCommentsPageEmbedded 40 8 TestListWisps +40 9 TestBumpEpochCarryingNeitherCarriesNorServesARowFromAFutureEpoch 40 9 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle 40 9 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent 40 9 TestEmbeddedDependencyEditorSnapshotsTheRequest @@ -237,6 +239,7 @@ 40 10 TestRunInTransactionPostCommitStageFailureIsIndeterminate 40 10 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient +40 11 TestABumpUnderOneStoreIDMovesTheEpochSeenThroughEveryStoreID 40 11 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges 40 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments 40 11 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches @@ -338,6 +341,7 @@ 40 19 TestRemoteVerbsPresentStoredPeerCredentials 40 19 TestRenameLabelMintsOneVersionPerTouchedBead +40 20 TestBumpEpochCarryingCarriesOnlyTheLineageHoldingTheNewestRow 40 20 TestCreateIssue 40 20 TestEmbeddedCheckedWrappersRouteTheirPreconditions 40 20 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps @@ -349,6 +353,7 @@ 40 20 TestHeartbeatRejectsWispEmbedded 40 20 TestIgnoredCursorHealNeverResurrectsRolledBackVersions +40 21 TestBumpEpochCarryingCarriesTheLiveRowsOfEveryStoreInTheDatabase 40 21 TestCreateRejectsCrossTableIDCollision 40 21 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards 40 21 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants @@ -435,6 +440,7 @@ 40 27 TestRemoteGitDataRefOnEngine 40 27 TestSweeperContract +40 28 TestBumpEpochCarryingDoesNotCarryALostLineage 40 28 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy 40 28 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed 40 28 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow @@ -472,6 +478,7 @@ 40 30 TestLegacyTrackedIgnoredCursorUnwedgesPull 40 30 TestUpdateMergeOps_DoltEngine_TwoIssues +40 31 TestBumpEpochCarryingWritesOneCarryRowPerLiveIDNamingTheLineageRoot 40 31 TestDualWriteContract 40 31 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget 40 31 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes @@ -484,6 +491,7 @@ 40 31 TestReadyCounterContract 40 31 TestWithPeerAuth_KeyMismatchFailsClosed +40 32 TestCurrentAddressForNeverReseedsAnAbsentStoreEpochRow 40 32 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone 40 32 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget 40 32 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers @@ -504,11 +512,13 @@ 40 33 TestEmbeddedRelationsResolvesTheAnchorIDExactly 40 33 TestEmbeddedSyncCommitsConfigBeforeMerge 40 33 TestEventsAutoPruneThrottleSkipsTheSecondPass +40 33 TestFirstMintLeavesTheEpochTablesCommitted 40 33 TestGetDependentRecordsEmbedded 40 33 TestPruneEventsBatchDeletesTheLowestSeqsOnly 40 33 TestTreeWalkerContract 40 34 TestAddLabel +40 34 TestBumpEpochCarryingLeavesTheEpochTablesCommitted 40 34 TestDecryptPassword_KeyMismatchNamesTheLocalKey 40 34 TestEmbeddedBlockedRecheckRecordsUnblockingWrites 40 34 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor @@ -532,6 +542,7 @@ 40 35 TestReadyWorkWithMissingDeferredParentTables 40 35 TestReclaimExpiredLeaseSurvivesRestartEmbedded +40 36 TestBumpEpochCarryingFailsAndRollsBackWhenARowAlreadyExistsAtTheNewEpoch 40 36 TestCommenterContract 40 36 TestConcurrentOpenReturnsCached 40 36 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory @@ -554,6 +565,7 @@ 40 37 TestEmbeddedMergeAndSettleReportsOperatorConflicts 40 37 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse 40 37 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch +40 37 TestEpochWritesReportAnUnpublishedCommitAsIndeterminate 40 37 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern 40 38 TestCreateIssues diff --git a/backend/conformance/retention_epoch_contract.go b/backend/conformance/retention_epoch_contract.go index 472b7a896..b2c0d100e 100644 --- a/backend/conformance/retention_epoch_contract.go +++ b/backend/conformance/retention_epoch_contract.go @@ -769,7 +769,21 @@ type EpochFixture struct { // Version now resolves to under the new epoch. A nil hook means this // backend cannot report that, and the case that needs it skips with // that reason. + // + // It only reads: it mints nothing and changes no state. It returns the + // address it was given when nothing carried the Version to a new one, + // and an error for an address the store does not serve. CurrentAddressFor func(ctx context.Context, storeID string, oldAddress Address) (Address, error) + + // LoseVersion records that storeID no longer serves the Version named by + // address. That address, and every address the same Version was carried + // to or from by a token-scheme change, resolves GoneReorganization from + // then on, in every later epoch: an epoch voids what the store lost, + // never what it still serves (R20-n). It is idempotent, never revives + // anything, and returns an error for an address storeID never minted. A + // nil LoseVersion means this backend cannot record a loss, and the cases + // that need one skip with that reason. + LoseVersion func(ctx context.Context, storeID string, address Address) error } // RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange pins R20-m: a @@ -819,94 +833,258 @@ func RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t *testing.T, ct // RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed pins R20-n — "Rev7 // tightening... the clause most likely to be gotten wrong" in the -// architecture doc's own words. A prior-epoch Address goes -// GoneReorganization UNLESS the backend still serves that Version, in which -// case it must stay Live AND additionally report the Address it now -// resolves to under the new epoch. +// architecture doc's own words: an epoch voids what the store lost, never +// what it still serves. // -// StillServes is an ORACLE here, not a control: this case cannot make a real -// backend keep serving one address and drop another, since Phase 0 wires no -// real backend. It asks StillServes what the fixture (once real) decided, -// and checks the rest of the contract's promise against that decision for -// both outcomes. +// Survival is RECORDED, never inferred (gastownhall/beads#6664, bee-ghosttrack +// review 5360880888, Major 1). An earlier form of this case made a Version +// survive by minting it again in every later epoch, and treated a Version +// nobody minted again as lost. That reads loss off the absence of a mint, so +// it could not express a Version the store lost after minting it, and it +// voided every survivor that nobody happened to mint again. Here a restore or +// a destructive reinit leaves every address row alone: an address the store +// still serves stays Live at its own address with no upkeep, and the only way +// an address goes Gone is LoseVersion saying so. +// +// Each phase is a named subtest, so a backend that cannot record a loss skips +// only the phases that need LoseVersion, and a failure names its phase. func RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t *testing.T, ctx context.Context, fixture EpochFixture) { t.Helper() - if fixture.MintUnderEpoch == nil { - t.Skip("this backend has no way to mint a Version under a known epoch (MintUnderEpoch is nil)") - } - if fixture.BumpEpoch == nil { - t.Skip("this backend has no epoch concept yet (BumpEpoch is nil)") - } - if fixture.StillServes == nil { - t.Skip("this backend cannot report whether it still serves a prior-epoch Version (StillServes is nil)") - } - if fixture.Resolve == nil { - t.Skip("this backend does not yet report retention state (Resolve is nil)") - } - if fixture.CurrentAddressFor == nil { - t.Skip("this backend cannot report a still-served Version's new Address (CurrentAddressFor is nil)") - } - store := epochStore(fixture, "voids") + epochSkipUnlessReady(t, fixture, false) - addrA, err := fixture.MintUnderEpoch(ctx, store, "record-a") - if err != nil { - t.Fatalf("MintUnderEpoch(record-a): %v", err) + // survivor-and-lost: of two Versions minted before a transition, the one + // the store loses goes Gone and the other stays Live at the address it + // already had, with no re-mint, for both triggers that leave addresses + // alone. + t.Run("survivor-and-lost", func(t *testing.T) { + if fixture.LoseVersion == nil { + t.Skip(epochLoseVersionSkip) + } + for _, trigger := range []EpochBumpTrigger{EpochBumpTriggerRestore, EpochBumpTriggerDestructiveReinit} { + t.Run(trigger.String(), func(t *testing.T) { + store := epochStore(fixture, "survivor-"+trigger.String()) + addrA := epochMintAddress(t, ctx, fixture, store, "record-a") + addrB := epochMintAddress(t, ctx, fixture, store, "record-b") + + newEpoch := epochBumpTo(t, ctx, fixture, store, trigger) + epochLoseAddress(t, ctx, fixture, store, addrB) + + epochRequireLive(t, ctx, fixture, store, addrA, newEpoch, + "record-a was never lost, so the transition leaves it served at its own address with no re-mint (R20-n's exception)") + epochRequireGone(t, ctx, fixture, store, addrB, newEpoch, + "record-b was lost, so the transition voids it (R20-n)") + + current := epochCurrentAddress(t, ctx, fixture, store, addrA) + epochRequireLive(t, ctx, fixture, store, current, newEpoch, + "the address CurrentAddressFor reports for a survivor must be served") + epochRequireLive(t, ctx, fixture, store, addrA, newEpoch, + "asking CurrentAddressFor must not retire or move the survivor's address") + + if lostCurrent, err := fixture.CurrentAddressFor(ctx, store, addrB); err == nil { + t.Errorf("CurrentAddressFor(a lost address) = %s with no error, want an error: a lost Version has no current address", lostCurrent) + } + epochRequireGone(t, ctx, fixture, store, addrB, newEpoch, + "CurrentAddressFor on a lost address must not bring it back") + }) + } + }) + + // gone-is-terminal: once an address is lost it stays lost. A later mint of + // the same id starts a new lineage under a new address, and neither that + // mint nor CurrentAddressFor revives the lost one (bee-ghosttrack's + // assertion for Major 1: a read-only lookup must not mint a replacement + // that makes a lost address look served). + t.Run("gone-is-terminal", func(t *testing.T) { + if fixture.LoseVersion == nil { + t.Skip(epochLoseVersionSkip) + } + store := epochStore(fixture, "terminal") + lost := epochMintAddress(t, ctx, fixture, store, "record-g") + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + epochLoseAddress(t, ctx, fixture, store, lost) + laterEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + + fresh := epochMintAddress(t, ctx, fixture, store, "record-g") + if fresh == lost { + t.Fatalf("MintUnderEpoch(record-g) returned %s, the address already lost: a lost address is never handed out again", lost) + } + epochRequireLive(t, ctx, fixture, store, fresh, laterEpoch, + "a fresh mint of the id is a new lineage and is served") + epochRequireGone(t, ctx, fixture, store, lost, laterEpoch, + "a later mint of the same id must not revive a lost address") + + if replacement, err := fixture.CurrentAddressFor(ctx, store, lost); err == nil { + t.Errorf("CurrentAddressFor(a lost address) = %s with no error, want an error: asking where a lost address resolves must not mint a replacement", replacement) + } + epochRequireGone(t, ctx, fixture, store, lost, laterEpoch, + "CurrentAddressFor on a lost address must not revive it") + + // Losing it a second time is a no-op that leaves the fresh lineage + // alone. + epochLoseAddress(t, ctx, fixture, store, lost) + epochRequireLive(t, ctx, fixture, store, fresh, laterEpoch, + "losing the old address again must leave the fresh lineage served") + }) + + // survival-needs-no-upkeep: with nothing lost, an address stays Live across + // any number of counter-only transitions without being minted again, and a + // later mint of the same id neither retires it nor revives anything. This + // is the phase the earlier, mint-driven form failed: it answered Gone for + // every survivor nobody minted again. + t.Run("survival-needs-no-upkeep", func(t *testing.T) { + store := epochStore(fixture, "upkeep") + kept := epochMintAddress(t, ctx, fixture, store, "record-c") + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerDestructiveReinit) + finalEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + + epochRequireLive(t, ctx, fixture, store, kept, finalEpoch, + "nothing was lost, so three counter-only transitions leave the address served without a re-mint") + + fresh := epochMintAddress(t, ctx, fixture, store, "record-c") + if fresh == kept { + t.Fatalf("MintUnderEpoch(record-c) returned %s again after three bumps, want a new address for the new epoch", kept) + } + epochRequireLive(t, ctx, fixture, store, kept, finalEpoch, + "a later mint of the same id must not retire a served address") + epochRequireLive(t, ctx, fixture, store, fresh, finalEpoch, + "the fresh mint is served") + + current := epochCurrentAddress(t, ctx, fixture, store, kept) + epochRequireLive(t, ctx, fixture, store, current, finalEpoch, + "the current address of a survivor must be served") + epochRequireLive(t, ctx, fixture, store, kept, finalEpoch, + "asking for the current address must not change the survivor") + }) +} + +// RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose pins R20-n's retained +// mapping for the one trigger that changes how addresses are encoded. A +// token-scheme-change bump gives every Version the store still serves a new +// address under the new epoch and keeps the old address resolving, so a holder +// of the old address is neither stranded nor pointed at something else. A +// Version the store already lost gets no new address and stays Gone. +func RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose(t *testing.T, ctx context.Context, fixture EpochFixture) { + t.Helper() + epochSkipUnlessReady(t, fixture, true) + store := epochStore(fixture, "scheme-change") + + addrA := epochMintAddress(t, ctx, fixture, store, "record-a") + addrB := epochMintAddress(t, ctx, fixture, store, "record-b") + // record-b is lost in the epoch it was minted in, which a store may do. + epochLoseAddress(t, ctx, fixture, store, addrB) + + newEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerTokenSchemeChange) + + epochRequireLive(t, ctx, fixture, store, addrA, newEpoch, + "record-a is still served after the scheme change, through the retained mapping") + epochRequireGone(t, ctx, fixture, store, addrB, newEpoch, + "record-b was lost before the scheme change, which must not carry it") + + carried := epochCurrentAddress(t, ctx, fixture, store, addrA) + if carried == addrA { + t.Errorf("CurrentAddressFor(%s) = the same address after a token-scheme change, want a new one: a scheme change re-encodes every served address, which is what tells a carry from a no-op", addrA) } - addrB, err := fixture.MintUnderEpoch(ctx, store, "record-b") - if err != nil { - t.Fatalf("MintUnderEpoch(record-b): %v", err) + epochRequireLive(t, ctx, fixture, store, carried, newEpoch, + "the carried address must be served under the new epoch") + epochRequireLive(t, ctx, fixture, store, addrA, newEpoch, + "the old address must keep resolving after its carried address is looked up") + + if again := epochCurrentAddress(t, ctx, fixture, store, carried); again != carried { + t.Errorf("CurrentAddressFor(%s) = %s, want the carried address itself: from the newest address the mapping is already current", carried, again) + } + if lostCurrent, err := fixture.CurrentAddressFor(ctx, store, addrB); err == nil { + t.Errorf("CurrentAddressFor(a lost address) = %s with no error, want an error: a lost Version is not carried", lostCurrent) + } +} + +// RunACarriedAddressKeepsResolvingAcrossLaterBumps pins that a carried mapping +// is recorded once and then simply persists: later counter-only transitions do +// not change it, and a second token-scheme change extends it to the newest +// address without breaking any older one. Every address of the Version points +// directly at the newest address (one hop from any member), never along a +// chain. +func RunACarriedAddressKeepsResolvingAcrossLaterBumps(t *testing.T, ctx context.Context, fixture EpochFixture) { + t.Helper() + epochSkipUnlessReady(t, fixture, false) + store := epochStore(fixture, "carried-later") + + addrA := epochMintAddress(t, ctx, fixture, store, "record-a") + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerTokenSchemeChange) + addrA1 := epochCurrentAddress(t, ctx, fixture, store, addrA) + if addrA1 == addrA { + t.Fatalf("CurrentAddressFor(%s) = the same address after a token-scheme change, want a new one", addrA) } - newEpoch, err := fixture.BumpEpoch(ctx, store, EpochBumpTriggerRestore) - if err != nil { - t.Fatalf("BumpEpoch: %v", err) + // Two counter-only transitions, with nothing minted in between. + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + counterEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerDestructiveReinit) + epochRequireLive(t, ctx, fixture, store, addrA, counterEpoch, + "the original address must still resolve after counter-only transitions") + epochRequireLive(t, ctx, fixture, store, addrA1, counterEpoch, + "the carried address must still resolve after counter-only transitions") + if got := epochCurrentAddress(t, ctx, fixture, store, addrA); got != addrA1 { + t.Errorf("CurrentAddressFor(%s) = %s, want %s: a counter-only transition does not change the mapping", addrA, got, addrA1) } - servesA, err := fixture.StillServes(ctx, store, addrA) - if err != nil { - t.Fatalf("StillServes(%s): %v", addrA, err) + // A second scheme change carries the Version again. + secondEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerTokenSchemeChange) + addrA2 := epochCurrentAddress(t, ctx, fixture, store, addrA) + if addrA2 == addrA || addrA2 == addrA1 { + t.Fatalf("CurrentAddressFor(%s) = %s after a second scheme change, want an address different from %s and %s", addrA, addrA2, addrA, addrA1) } - servesB, err := fixture.StillServes(ctx, store, addrB) - if err != nil { - t.Fatalf("StillServes(%s): %v", addrB, err) + for _, member := range []Address{addrA, addrA1, addrA2} { + epochRequireLive(t, ctx, fixture, store, member, secondEpoch, + "every address of a carried Version must resolve after a second scheme change") } - if servesA == servesB { - t.Skip("this scenario needs one still-served and one no-longer-served address to exercise both halves of R20-n; StillServes reported the same answer for both, so this fixture gives this case nothing to contrast") + if got := epochCurrentAddress(t, ctx, fixture, store, addrA1); got != addrA2 { + t.Errorf("CurrentAddressFor(%s) = %s, want %s: every address of the Version points straight at the newest one", addrA1, got, addrA2) } - stillServed, noLongerServed := addrA, addrB - if servesB && !servesA { - stillServed, noLongerServed = addrB, addrA + finalEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + epochRequireLive(t, ctx, fixture, store, addrA2, finalEpoch, + "the newest carried address must still resolve after a later restore") +} + +// RunALostAddressStaysGoneAcrossASchemeChange pins that loss is terminal even +// across a token-scheme change. Losing a Version loses every address of its +// lineage at once, and a later scheme change must not carry a lost lineage +// back into service. +func RunALostAddressStaysGoneAcrossASchemeChange(t *testing.T, ctx context.Context, fixture EpochFixture) { + t.Helper() + epochSkipUnlessReady(t, fixture, true) + store := epochStore(fixture, "lost-scheme-change") + + addrA := epochMintAddress(t, ctx, fixture, store, "record-a") + epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerTokenSchemeChange) + addrA1 := epochCurrentAddress(t, ctx, fixture, store, addrA) + lostEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerRestore) + + // One lineage, two addresses: losing either loses both. + epochLoseAddress(t, ctx, fixture, store, addrA) + epochRequireGone(t, ctx, fixture, store, addrA, lostEpoch, + "the original address of a lost Version must be Gone") + epochRequireGone(t, ctx, fixture, store, addrA1, lostEpoch, + "the carried address of a lost Version must be Gone too") + + schemeEpoch := epochBumpTo(t, ctx, fixture, store, EpochBumpTriggerTokenSchemeChange) + epochRequireGone(t, ctx, fixture, store, addrA, schemeEpoch, + "a later scheme change must not carry a lost lineage back into service") + epochRequireGone(t, ctx, fixture, store, addrA1, schemeEpoch, + "a later scheme change must not carry a lost lineage back into service") + for _, lostAddress := range []Address{addrA, addrA1} { + if current, err := fixture.CurrentAddressFor(ctx, store, lostAddress); err == nil { + t.Errorf("CurrentAddressFor(%s) = %s with no error, want an error: a lost lineage has no current address", lostAddress, current) + } } - voidedAnswer, err := fixture.Resolve(ctx, store, noLongerServed) - if err != nil { - t.Fatalf("Resolve(no-longer-served): %v", err) - } - if voidedAnswer.Restriction != RestrictionGoneReorganization { - t.Errorf("Resolve(a prior-epoch Address no longer served) = %s, want RestrictionGoneReorganization (R20-n)", voidedAnswer.Restriction) - } - if voidedAnswer.Epoch == nil { - t.Fatal("Resolve(a prior-epoch Address no longer served).Epoch = nil, want the current epoch populated (R20-n: the answer must name the current epoch, not just Restriction alone)") - } else if *voidedAnswer.Epoch != newEpoch { - t.Errorf("Resolve(a prior-epoch Address no longer served).Epoch = %d, want the current epoch %d", *voidedAnswer.Epoch, newEpoch) - } - - keptAnswer, err := fixture.Resolve(ctx, store, stillServed) - if err != nil { - t.Fatalf("Resolve(still-served): %v", err) - } - if keptAnswer.Restriction != RestrictionLive { - t.Errorf("Resolve(a prior-epoch Address the backend still serves) = %s, want RestrictionLive (R20-n's exception)", keptAnswer.Restriction) - } - - newAddr, err := fixture.CurrentAddressFor(ctx, store, stillServed) - if err != nil { - t.Fatalf("CurrentAddressFor(%s): %v", stillServed, err) - } - if newAddr == "" { - t.Error("CurrentAddressFor(a still-served prior-epoch Address) returned an empty Address, want a real one under the new epoch") + fresh := epochMintAddress(t, ctx, fixture, store, "record-a") + if fresh == addrA || fresh == addrA1 { + t.Fatalf("MintUnderEpoch(record-a) returned %s, an address of the lost lineage: a lost address is never handed out again", fresh) } + epochRequireLive(t, ctx, fixture, store, fresh, schemeEpoch, + "a fresh mint of the id after the loss is a new lineage and is served") } // --- fixture helpers ------------------------------------------------------- @@ -953,6 +1131,137 @@ func epochStore(fixture EpochFixture, tag string) string { return fixture.IssuePrefix + "-epoch-store-" + tag } +// epochLoseVersionSkip is the skip reason of every phase that needs +// EpochFixture.LoseVersion. +const epochLoseVersionSkip = "this backend cannot record that the store lost a Version (LoseVersion is nil)" + +// epochSkipUnlessReady skips the case when the fixture lacks a capability the +// epoch cases need. The cases that record a loss also need LoseVersion. The +// all-nil wiring test runs every case against an empty fixture, so each case +// must call this before it touches any of them. +func epochSkipUnlessReady(t *testing.T, fixture EpochFixture, needsLoseVersion bool) { + t.Helper() + if fixture.MintUnderEpoch == nil { + t.Skip("this backend has no way to mint a Version under a known epoch (MintUnderEpoch is nil)") + } + if fixture.BumpEpoch == nil { + t.Skip("this backend has no epoch concept yet (BumpEpoch is nil)") + } + if fixture.StillServes == nil { + t.Skip("this backend cannot report whether it still serves a prior-epoch Version (StillServes is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.CurrentAddressFor == nil { + t.Skip("this backend cannot report a still-served Version's new Address (CurrentAddressFor is nil)") + } + if needsLoseVersion && fixture.LoseVersion == nil { + t.Skip(epochLoseVersionSkip) + } +} + +// epochMintAddress mints id on store, failing the case if minting fails. +func epochMintAddress(t *testing.T, ctx context.Context, fixture EpochFixture, store, id string) Address { + t.Helper() + address, err := fixture.MintUnderEpoch(ctx, store, id) + if err != nil { + t.Fatalf("MintUnderEpoch(%s, %s): %v", store, id, err) + } + return address +} + +// epochBumpTo bumps store's epoch for trigger and returns the epoch it +// reports. Epochs are captured from the return value, never assumed: the +// counter is one per database, shared by every store the suite uses. +func epochBumpTo(t *testing.T, ctx context.Context, fixture EpochFixture, store string, trigger EpochBumpTrigger) int { + t.Helper() + epoch, err := fixture.BumpEpoch(ctx, store, trigger) + if err != nil { + t.Fatalf("BumpEpoch(%s, %s): %v", store, trigger, err) + } + return epoch +} + +// epochLoseAddress records that store lost the Version behind address. +func epochLoseAddress(t *testing.T, ctx context.Context, fixture EpochFixture, store string, address Address) { + t.Helper() + if err := fixture.LoseVersion(ctx, store, address); err != nil { + t.Fatalf("LoseVersion(%s, %s): %v", store, address, err) + } +} + +// epochCurrentAddress asks where address now resolves, failing the case on an +// error or an empty answer. +func epochCurrentAddress(t *testing.T, ctx context.Context, fixture EpochFixture, store string, address Address) Address { + t.Helper() + current, err := fixture.CurrentAddressFor(ctx, store, address) + if err != nil { + t.Fatalf("CurrentAddressFor(%s, %s): %v", store, address, err) + } + if current == "" { + t.Fatalf("CurrentAddressFor(%s, %s) returned an empty Address, want a real one", store, address) + } + return current +} + +// epochRequireLive asserts that address is served at epoch: StillServes is true +// and Resolve answers RestrictionLive naming that epoch. why says what the +// assertion is checking, for the failure message. +func epochRequireLive(t *testing.T, ctx context.Context, fixture EpochFixture, store string, address Address, epoch int, why string) { + t.Helper() + serves, err := fixture.StillServes(ctx, store, address) + if err != nil { + t.Fatalf("StillServes(%s): %v", address, err) + } + if !serves { + t.Errorf("StillServes(%s) = false, want true: %s", address, why) + } + answer, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s): %v", address, err) + } + if answer.Restriction != RestrictionLive { + t.Errorf("Resolve(%s) = %s, want RestrictionLive: %s", address, answer.Restriction, why) + } + epochRequireAnswerEpoch(t, answer, address, epoch, why) +} + +// epochRequireGone asserts that address is no longer served at epoch: +// StillServes is false and Resolve answers RestrictionGoneReorganization +// naming that epoch. +func epochRequireGone(t *testing.T, ctx context.Context, fixture EpochFixture, store string, address Address, epoch int, why string) { + t.Helper() + serves, err := fixture.StillServes(ctx, store, address) + if err != nil { + t.Fatalf("StillServes(%s): %v", address, err) + } + if serves { + t.Errorf("StillServes(%s) = true, want false: %s", address, why) + } + answer, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s): %v", address, err) + } + if answer.Restriction != RestrictionGoneReorganization { + t.Errorf("Resolve(%s) = %s, want RestrictionGoneReorganization: %s", address, answer.Restriction, why) + } + epochRequireAnswerEpoch(t, answer, address, epoch, why) +} + +// epochRequireAnswerEpoch asserts that a Live or Gone answer names the current +// epoch (R20-n: the answer carries the epoch, not just the Restriction). +func epochRequireAnswerEpoch(t *testing.T, answer RetentionAnswer, address Address, epoch int, why string) { + t.Helper() + if answer.Epoch == nil { + t.Errorf("Resolve(%s).Epoch = nil, want the current epoch %d: %s", address, epoch, why) + return + } + if *answer.Epoch != epoch { + t.Errorf("Resolve(%s).Epoch = %d, want the current epoch %d: %s", address, *answer.Epoch, epoch, why) + } +} + // sameRetentionAnswer compares two RetentionAnswer values by content rather // than identity, so a case can assert "this did not change" without // depending on the concrete Restriction the backend happened to pick. diff --git a/backend/conformance/versioned_history_wiring_test.go b/backend/conformance/versioned_history_wiring_test.go index 08eed2b53..b022b716e 100644 --- a/backend/conformance/versioned_history_wiring_test.go +++ b/backend/conformance/versioned_history_wiring_test.go @@ -129,5 +129,14 @@ func TestVersionedHistoryPhase0RunsInFullSkipMode(t *testing.T) { t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) }) + t.Run("TokenSchemeChangeCarriesLiveAddressesAndOnlyThose", func(t *testing.T) { + RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose(t, ctx, fixture) + }) + t.Run("ACarriedAddressKeepsResolvingAcrossLaterBumps", func(t *testing.T) { + RunACarriedAddressKeepsResolvingAcrossLaterBumps(t, ctx, fixture) + }) + t.Run("ALostAddressStaysGoneAcrossASchemeChange", func(t *testing.T) { + RunALostAddressStaysGoneAcrossASchemeChange(t, ctx, fixture) + }) }) } diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 4be1fec0b..773a53d6b 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "dependency_editor.go", "edgereader.go", "ephemeral_routing.go", + "epoch_cas.go", "errors.go", "events.go", "federation.go", @@ -221,6 +222,7 @@ go_test( "edge_reader_contract_test.go", "empty_commit_test.go", "ephemeral_routing_test.go", + "epoch_cas_publish_test.go", "errors_test.go", "event_commit_boundary_test.go", "events_since_test.go", diff --git a/internal/storage/dolt/epoch_cas.go b/internal/storage/dolt/epoch_cas.go new file mode 100644 index 000000000..e455fda27 --- /dev/null +++ b/internal/storage/dolt/epoch_cas.go @@ -0,0 +1,248 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + + storeops "github.com/steveyegge/beads/internal/storage/issueops" +) + +// CurrentEpoch, BumpEpoch, BumpEpochCarrying, MintUnderEpoch, LoseVersion, +// StillServes, Resolve and CurrentAddressFor give this leg R20's +// epoch-transition enforcement +// (gastownhall/beads#5898 revision 9, #6136), +// backed by store_epoch (migration 0067) and epoch_minted_addresses +// (migration 0072). +// +// UNLIKE MetadataCAS, THIS ROLE HAS NO PUBLIC issueops INTERFACE: the +// conformance contract's own seven epoch hooks are bare function types (see +// backend/conformance.EpochFixture) — so there is no role accessor to +// return one of, and these are direct methods on *DoltStore instead of a +// wrapper struct. +// +// These also have no domain-repository exposure: the unit-of-work leg +// reaches internal/storage/issueops's epoch Tx functions directly rather +// than through the domain issue repository (see internal/storage/uow's own +// epoch wiring), so nothing forces these types out of issueops into the +// root storage package. Method signatures here use plain types plus +// storeops.EpochResolveResult directly; backend/conformance's own +// Address/EpochBumpTrigger/RetentionAnswer vocabulary is translated only in +// this leg's contract-test file, per internal/storage/issueops/epoch_cas.go's +// header. +// +// Method names match backend/conformance.EpochFixture's own field names +// verbatim. +// +// PUBLICATION ORDER. The four writers +// commit their SQL transaction FIRST and stage and Dolt-commit afterwards, +// never from inside the open transaction. doltAddAndCommitInTx builds the Dolt +// commit from the transaction's BEGIN-time snapshot, so under concurrent +// writers it writes every row changed in the meantime back to its old value, +// and store_epoch and epoch_minted_addresses are append-only: a row lost that +// way is never rewritten (the HAZARD block on that helper). +// +// AN ERROR RETURNED AFTER THE SQL COMMIT MUST NOT BE REPLAYED. Every error a +// writer returns once its SQL commit has landed, or may have, wraps +// ErrCommitIndeterminate, the storage-wide no-replay sentinel: a replayed bump +// advances the counter a second time. A bump returns the epoch it committed, or +// would have, with that error, and a mint its address; the caller learns which +// by reading CurrentEpoch or StillServes, never by writing again. The embedded +// leg does the same. + +// CurrentEpoch reports storeID's current epoch generation. +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +func (s *DoltStore) CurrentEpoch(ctx context.Context, storeID string) (int, error) { + var epoch int + if err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + epoch, err = storeops.CurrentEpochInTx(ctx, tx, storeID) + return err + }); err != nil { + return 0, err + } + return epoch, nil +} + +// BumpEpoch advances storeID's epoch generation by one for reason. +// +// A Dolt commit that fails after the SQL commit is RETURNED, where the mint +// paths below log it and carry on: a bump is rare and +// operator-driven, and its caller (the restore trigger) must not report a +// restore complete while the bump is committed but unpublished. In that case +// the bump HAS applied: the epoch returned is the new one, the row is durable +// in the working set and rides the next Dolt commit, and the error wraps +// ErrCommitIndeterminate so that nothing bumps again to retry. A SQL commit +// whose result was lost is returned the same way, with the epoch it would have +// committed (failedBump). +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +func (s *DoltStore) BumpEpoch(ctx context.Context, storeID, reason string) (int, error) { + var epoch int + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + var err error + epoch, err = storeops.BumpEpochInTx(ctx, tx, storeID, reason) + return err + }); err != nil { + return failedBump(storeID, epoch, err) + } + if err := s.doltAddAndCommitPostTx(ctx, []string{"store_epoch"}, + fmt.Sprintf("bd: bump epoch for %s (%s)", storeID, reason)); err != nil { + return epoch, unpublishedBump(storeID, epoch, err) + } + return epoch, nil +} + +// failedBump is what a bump returns when its SQL transaction reports an error. +// An error wrapping ErrCommitIndeterminate came from the commit of a body that +// ran to completion, so the commit may have landed: the epoch that body set is +// returned with it, and the caller reads CurrentEpoch to learn which. Any other +// error means nothing landed. +func failedBump(storeID string, epoch int, err error) (int, error) { + if errors.Is(err, ErrCommitIndeterminate) { + return epoch, fmt.Errorf("epoch bump for %s may have applied (epoch %d): %w", storeID, epoch, err) + } + return 0, err +} + +// unpublishedBump is the error for a bump whose SQL commit landed and whose +// Dolt commit then failed. It wraps ErrCommitIndeterminate whatever the cause: +// doltAddAndCommitPostTx returns some failures without it (a staged-set read +// that fails, a breaker that refuses the commit), and those causes are no more +// replayable than the rest once the counter has moved. +func unpublishedBump(storeID string, epoch int, err error) error { + return fmt.Errorf("epoch bump for %s applied (epoch %d) but its Dolt commit failed: %w: %w", storeID, epoch, err, ErrCommitIndeterminate) +} + +// BumpEpochCarrying is the token-scheme-change bump: BumpEpoch, and also the +// carry of every live address in the database to the new epoch +// (storeops.BumpEpochCarryingInTx). It writes store_epoch and +// epoch_minted_addresses, so it publishes both. It surfaces a failed Dolt +// commit exactly as BumpEpoch does, and the bump has applied in that case: the +// epoch returned is the new one and the caller must not bump again to retry. +// A lost SQL commit result is returned as BumpEpoch returns it (failedBump). +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +// +// CALLERS MUST HOLD THE WORKSPACE GATE (internal/workspacegate) EXCLUSIVELY FOR +// THE BUMP, as a restore does: see storeops.BumpEpochCarryingInTx. +func (s *DoltStore) BumpEpochCarrying(ctx context.Context, storeID, reason string) (int, error) { + var epoch int + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + var err error + epoch, err = storeops.BumpEpochCarryingInTx(ctx, tx, storeID, reason) + return err + }); err != nil { + return failedBump(storeID, epoch, err) + } + if err := s.doltAddAndCommitPostTx(ctx, []string{"store_epoch", "epoch_minted_addresses"}, + fmt.Sprintf("bd: carry addresses to epoch %d for %s (%s)", epoch, storeID, reason)); err != nil { + return epoch, unpublishedBump(storeID, epoch, err) + } + return epoch, nil +} + +// MintUnderEpoch mints id's address under storeID's current epoch. +// +// It publishes like every issue mutation, through +// runIssueOperationTxWithMessage: a Dolt commit that fails after the SQL commit +// is logged, not returned. The mint is durable in the working set and rides the +// next Dolt commit, and mints are frequent and automated, so failing one over a +// missing history commit would only turn a harmless gap into caller errors. A +// SQL commit whose result was lost is returned, wrapping +// ErrCommitIndeterminate, together with the address the mint may have +// committed; StillServes says whether it did. +// +// A mint whose transaction overlaps a BumpEpoch reads the epoch before the bump +// and commits an address minted under the epoch the bump replaced. That address +// is a Live survivor: a restore or reinit bump writes no address rows, and an +// address's status is read from its own row. TestMintOverlappingABumpLeavesALiveSurvivor +// records that interleaving, and TestAMintAfterAConcurrentLossNeverRevivesTheLostAddress +// the one that must never bring a lost address back. +func (s *DoltStore) MintUnderEpoch(ctx context.Context, storeID, id string) (string, error) { + var address string + if err := s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + address, err = storeops.MintUnderEpochInTx(ctx, tx, storeID, id) + if err != nil { + return nil, "", err + } + return storeops.EpochMintDirtyTables(), + fmt.Sprintf("bd: mint %s under epoch for %s", id, storeID), nil + }); err != nil { + if errors.Is(err, ErrCommitIndeterminate) { + return address, err + } + return "", err + } + return address, nil +} + +// StillServes reports whether address is still served under storeID's +// current epoch. +func (s *DoltStore) StillServes(ctx context.Context, storeID, address string) (bool, error) { + var serves bool + if err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + serves, err = storeops.StillServesInTx(ctx, tx, storeID, address) + return err + }); err != nil { + return false, err + } + return serves, nil +} + +// Resolve answers R20's epoch-only restriction for address. +func (s *DoltStore) Resolve(ctx context.Context, storeID, address string) (storeops.EpochResolveResult, error) { + var result storeops.EpochResolveResult + if err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, err = storeops.ResolveEpochInTx(ctx, tx, storeID, address) + return err + }); err != nil { + return storeops.EpochResolveResult{}, err + } + return result, nil +} + +// CurrentAddressFor reports the address oldAddress now resolves to: the newest +// address of its lineage, or oldAddress itself when nothing carried it. It only +// reads, so it runs on this leg's read path and publishes nothing. +func (s *DoltStore) CurrentAddressFor(ctx context.Context, storeID, oldAddress string) (string, error) { + var address string + if err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + address, err = storeops.CurrentAddressForInTx(ctx, tx, storeID, oldAddress) + return err + }); err != nil { + return "", err + } + return address, nil +} + +// LoseVersion records that storeID no longer serves the Version named by +// address (storeops.LoseVersionInTx). It writes epoch_minted_addresses, so it +// publishes that table, and unlike a mint it RETURNS a failed Dolt commit: a +// loss its caller believes recorded must not be left unpublished. The loss has +// applied in that case, and the error wraps ErrCommitIndeterminate like a +// bump's, although replaying a loss is a no-op. +func (s *DoltStore) LoseVersion(ctx context.Context, storeID, address string) error { + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + return storeops.LoseVersionInTx(ctx, tx, storeID, address) + }); err != nil { + return err + } + if err := s.doltAddAndCommitPostTx(ctx, []string{"epoch_minted_addresses"}, + fmt.Sprintf("bd: lose version %s for %s", address, storeID)); err != nil { + return fmt.Errorf("loss of %s for %s recorded but its Dolt commit failed: %w: %w", address, storeID, err, ErrCommitIndeterminate) + } + return nil +} diff --git a/internal/storage/dolt/epoch_cas_publish_test.go b/internal/storage/dolt/epoch_cas_publish_test.go new file mode 100644 index 000000000..3928b1059 --- /dev/null +++ b/internal/storage/dolt/epoch_cas_publish_test.go @@ -0,0 +1,949 @@ +package dolt + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + mysql "github.com/go-sql-driver/mysql" + + storeops "github.com/steveyegge/beads/internal/storage/issueops" +) + +// These tests pin WHERE and HOW this leg publishes the R20 epoch tables +// (store_epoch, epoch_minted_addresses) to Dolt history +// (gastownhall/beads#6664). +// +// The first group runs on the sequence-capturing driver from +// dolt_commit_ordering_test.go, so it needs no running Dolt server: the +// ordering contract and the staged-table lists are properties of the SQL +// this leg sends, not of Dolt. The tests after them run against a real server: +// the first reproduces bee-ghosttrack's own probe (dolt_status after the first +// mint), the next three apply it to the carrying bump, to a recorded loss and to +// the read-only CurrentAddressFor, and the last three choreograph how a mint +// behaves beside a concurrent bump, beside a concurrent loss, and beside another +// transaction's mint and loss of its own address on that server. + +const epochPublishStoreID = "epoch-store" +const epochPublishMintedID = "record-a" + +// epochPublishAddress is the address of epochPublishMintedID under epoch 1, the +// one the fake serves as minted. It is minted on a fake of its own rather than +// written by hand: the fake serves a row at any address, so a hand-written one +// would go stale unnoticed if the encoding moved +// (TestEpochCASTestsBuildEveryAddressWithEpochAddress in issueops scans this +// file for one). +func epochPublishAddress() string { + f := newEpochFake() + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + address, err := (&DoltStore{db: db}).MintUnderEpoch(context.Background(), epochPublishStoreID, epochPublishMintedID) + if err != nil { + panic(fmt.Sprintf("mint %s on a fresh epoch fake: %v", epochPublishMintedID, err)) + } + return address +} + +// epochFake is a *DoltStore's driver for the epoch methods: it serves the +// reads the shared epoch Tx functions make, keeps the singleton's epoch moving +// when the bump's UPDATE runs, and can refuse every DOLT_COMMIT. +type epochFake struct { + rec *seqRecorder + connector *seqConnector + epoch atomic.Int64 + // publishErr, when set before the call under test, fails every + // CALL DOLT_COMMIT: the SQL transaction is fine, only publication is not. + publishErr error + // stagedReadErr, when set, fails the staged-set read stageAndCommit makes + // between DOLT_ADD and DOLT_COMMIT, a failure it returns without + // ErrCommitIndeterminate because nothing has reached Dolt history yet. + stagedReadErr error +} + +func newEpochFake() *epochFake { + f := &epochFake{rec: &seqRecorder{}} + f.epoch.Store(1) + f.connector = &seqConnector{rec: f.rec} + f.connector.queryErr = func(query string) error { + switch { + case strings.Contains(query, "UPDATE store_epoch"): + f.epoch.Add(1) + case strings.Contains(query, "DOLT_COMMIT") && f.publishErr != nil: + return f.publishErr + case strings.Contains(query, "FROM dolt_status WHERE staged") && f.stagedReadErr != nil: + return f.stagedReadErr + } + return nil + } + f.connector.rows = func(query string) (driver.Rows, bool) { + switch { + case strings.Contains(query, "FROM store_epoch"): + return &valRows{cols: []string{"epoch"}, vals: [][]driver.Value{{f.epoch.Load()}}}, true + case strings.Contains(query, "FROM epoch_minted_addresses WHERE address"): + // Every address is already minted, so CurrentAddressFor has an + // old address to look up and MintUnderEpoch has nothing to insert. + // A read that asks for the survival columns gets a Live row nothing + // carried; a read that does not gets the three-column shape. + if strings.Contains(query, "carried_from") { + return &valRows{ + cols: []string{"store_id", "minted_id", "minted_epoch", "carried_from", "gone_at_epoch"}, + vals: [][]driver.Value{{epochPublishStoreID, epochPublishMintedID, int64(1), nil, nil}}, + }, true + } + return &valRows{ + cols: []string{"store_id", "minted_id", "minted_epoch"}, + vals: [][]driver.Value{{epochPublishStoreID, epochPublishMintedID, int64(1)}}, + }, true + } + return nil, false + } + return f +} + +// epochArgConnector is the sequence-capturing connector plus the table each +// DOLT_ADD stages, which the sequence recorder drops. +type epochArgConnector struct{ *seqConnector } + +func (c epochArgConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.seqConnector.Connect(ctx) + if err != nil { + return nil, err + } + return &epochArgConn{seqConn: conn.(*seqConn)}, nil +} + +type epochArgConn struct{ *seqConn } + +func (c *epochArgConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if strings.Contains(query, "DOLT_ADD") && len(args) == 1 { + c.parent.rec.add(c.id, fmt.Sprintf("STAGE %v", args[0].Value)) + } + return c.seqConn.QueryContext(ctx, query, args) +} + +// epochPublishCall is one epoch method that writes. +type epochPublishCall struct { + name string + call func(ctx context.Context, s *DoltStore) error + // staged is the exact set of tables the call stages for its Dolt commit, + // sorted. + staged []string + // logsPublishFailure is true for the call that carries on past a failed Dolt + // commit; every other call returns it. + logsPublishFailure bool +} + +func epochPublishCalls() []epochPublishCall { + return []epochPublishCall{ + { + name: "BumpEpoch", + staged: []string{"store_epoch"}, + call: func(ctx context.Context, s *DoltStore) error { + _, err := s.BumpEpoch(ctx, epochPublishStoreID, "restore") + return err + }, + }, + { + name: "BumpEpochCarrying", + staged: []string{"epoch_minted_addresses", "store_epoch"}, + call: func(ctx context.Context, s *DoltStore) error { + _, err := s.BumpEpochCarrying(ctx, epochPublishStoreID, "token-scheme-change") + return err + }, + }, + { + name: "MintUnderEpoch", + staged: []string{"epoch_minted_addresses", "store_epoch"}, + logsPublishFailure: true, + call: func(ctx context.Context, s *DoltStore) error { + _, err := s.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID) + return err + }, + }, + { + name: "LoseVersion", + staged: []string{"epoch_minted_addresses"}, + call: func(ctx context.Context, s *DoltStore) error { + return s.LoseVersion(ctx, epochPublishStoreID, epochPublishAddress()) + }, + }, + } +} + +func epochPublishIndex(events []string, pred func(string) bool) int { + for i, ev := range events { + if pred(ev) { + return i + } + } + return -1 +} + +// TestEpochWritesPublishToDoltOnlyAfterTheSQLCommit is the ordering contract +// of dolt_commit_ordering_test.go applied to the epoch methods. Staging and +// Dolt-committing store_epoch or epoch_minted_addresses inside the still-open +// SQL transaction is the lost-update hazard documented on doltAddAndCommitInTx +// (LatentLabsSpace/NEXUS#92), and it costs more here than on issues: both +// tables are append-only, so a row reverted to its BEGIN-time value is never +// rewritten. +func TestEpochWritesPublishToDoltOnlyAfterTheSQLCommit(t *testing.T) { + for _, tc := range epochPublishCalls() { + t.Run(tc.name, func(t *testing.T) { + f := newEpochFake() + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + if err := tc.call(context.Background(), &DoltStore{db: db}); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + + events := f.rec.snapshot() + commitIdx := epochPublishIndex(events, func(ev string) bool { return ev == "COMMIT" }) + addIdx := epochPublishIndex(events, func(ev string) bool { return strings.Contains(ev, "DOLT_ADD") }) + doltCommitIdx := epochPublishIndex(events, func(ev string) bool { return strings.Contains(ev, "DOLT_COMMIT") }) + if commitIdx == -1 || addIdx == -1 || doltCommitIdx == -1 { + t.Fatalf("%s: want a SQL COMMIT, a DOLT_ADD and a DOLT_COMMIT; events: %v", tc.name, events) + } + if addIdx < commitIdx { + t.Errorf("%s: DOLT_ADD ran INSIDE the open transaction (index %d < COMMIT index %d) — lost-update ordering hazard; events: %v", + tc.name, addIdx, commitIdx, events) + } + if doltCommitIdx < commitIdx { + t.Errorf("%s: DOLT_COMMIT ran INSIDE the open transaction (index %d < COMMIT index %d) — lost-update ordering hazard; events: %v", + tc.name, doltCommitIdx, commitIdx, events) + } + }) + } +} + +// TestEpochWritesStageExactlyTheTablesTheyChange is the server-leg half of +// gastownhall/beads#6664 Major 2 (bee-ghosttrack, review 5360880888): the +// first mint on a store lazily inserts store_epoch's singleton +// (ensureStoreEpochRow), so a mint that stages only epoch_minted_addresses +// leaves the epoch row modified and uncommitted, and the Dolt commit holds +// minted rows without the epoch row they depend on. It states the rule for +// every writer, with exact sets: a restore or reinit bump changes only the +// counter, the carrying bump and a mint change both tables, and a recorded loss +// changes only address rows. A writer that stages less leaves rows behind in +// the working set, and one that stages more commits rows it did not write. +func TestEpochWritesStageExactlyTheTablesTheyChange(t *testing.T) { + for _, tc := range epochPublishCalls() { + t.Run(tc.name, func(t *testing.T) { + f := newEpochFake() + db := sql.OpenDB(epochArgConnector{f.connector}) + defer func() { _ = db.Close() }() + + if err := tc.call(context.Background(), &DoltStore{db: db}); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + + var staged []string + for _, ev := range f.rec.snapshot() { + if table, ok := strings.CutPrefix(ev, "STAGE "); ok { + staged = append(staged, table) + } + } + slices.Sort(staged) + staged = slices.Compact(staged) + if !slices.Equal(staged, tc.staged) { + t.Errorf("%s staged %v, want exactly %v (Major 2: a table the call writes must be committed with it, and one it does not write must not be)", tc.name, staged, tc.staged) + } + }) + } +} + +// TestBumpEpochSurfacesAPublicationFailureAfterTheSQLCommit pins how a bump +// reports a failed publication: a bump whose SQL transaction +// committed but whose Dolt commit failed must tell its caller, because a +// restore trigger must not report a restore complete over an unpublished +// bump. The bump itself must survive: it is durable in the working set, so a +// failed history commit is an error to report, never a reason to roll back. +func TestBumpEpochSurfacesAPublicationFailureAfterTheSQLCommit(t *testing.T) { + f := newEpochFake() + f.publishErr = errors.New("dolt commit refused") + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + epoch, err := (&DoltStore{db: db}).BumpEpoch(context.Background(), epochPublishStoreID, "restore") + + requirePublicationFailureReported(t, "BumpEpoch", f, err, f.publishErr) + if epoch != 2 { + t.Errorf("BumpEpoch returned epoch %d with the publication error, want 2: the bump applied, the error only says its Dolt commit did not", epoch) + } +} + +// requirePublicationFailureReported checks what a writer that reports a failed +// Dolt commit must show once its SQL transaction committed: the transaction was +// committed and never rolled back, and the error it returned wraps the failure +// (cause) and ErrCommitIndeterminate, so that nothing replays a write that has +// already applied. +func requirePublicationFailureReported(t *testing.T, name string, f *epochFake, err, cause error) { + t.Helper() + events := f.rec.snapshot() + if !slices.Contains(events, "COMMIT") || slices.Contains(events, "ROLLBACK") { + t.Errorf("%s: a failed Dolt commit must not undo the write: want the SQL transaction COMMITTED and never rolled back; events: %v", name, events) + } + if err == nil { + t.Fatalf("%s = nil error although its Dolt commit failed: the caller would report success for a write that is committed but unpublished; events: %v", name, events) + } + if !errors.Is(err, cause) { + t.Errorf("%s error = %v, want it to wrap the Dolt commit failure %v", name, err, cause) + } + if !errors.Is(err, ErrCommitIndeterminate) { + t.Errorf("%s error = %v, want it to wrap ErrCommitIndeterminate: the write has applied, and a caller that takes the error for a write that did not would replay it", name, err) + } +} + +// TestBumpEpochCarryingSurfacesAPublicationFailureAfterTheSQLCommit holds the +// carrying bump to the rule BumpEpoch is held to above. It writes the address +// rows of every live Version as well as the counter, and a token-scheme change +// must not report success over a bump that is committed but unpublished. The +// bump applied, so the epoch comes back with the error and the caller must not +// bump again to retry. +func TestBumpEpochCarryingSurfacesAPublicationFailureAfterTheSQLCommit(t *testing.T) { + f := newEpochFake() + f.publishErr = errors.New("dolt commit refused") + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + epoch, err := (&DoltStore{db: db}).BumpEpochCarrying(context.Background(), epochPublishStoreID, "token-scheme-change") + + requirePublicationFailureReported(t, "BumpEpochCarrying", f, err, f.publishErr) + if epoch != 2 { + t.Errorf("BumpEpochCarrying returned epoch %d with the publication error, want 2: the bump applied, the error only says its Dolt commit did not", epoch) + } +} + +// TestLoseVersionSurfacesAPublicationFailureAfterTheSQLCommit pins the same rule +// for a recorded loss: a loss its caller believes recorded must not be left +// unpublished. Unlike a mint, a loss is rare and deliberate, so a failed Dolt +// commit is an error to report, never one to log. +func TestLoseVersionSurfacesAPublicationFailureAfterTheSQLCommit(t *testing.T) { + f := newEpochFake() + f.publishErr = errors.New("dolt commit refused") + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + err := (&DoltStore{db: db}).LoseVersion(context.Background(), epochPublishStoreID, epochPublishAddress()) + + requirePublicationFailureReported(t, "LoseVersion", f, err, f.publishErr) +} + +// TestEpochWritesMarkAFailedStagedReadIndeterminate holds the writers that +// return a failed publication to the no-replay rule for the failure +// stageAndCommit itself returns as a plain error: its staged-set read before +// DOLT_COMMIT. Nothing has reached Dolt history at that point, but the write's +// SQL commit has landed, so a bump replayed on that error would advance the +// counter a second time. The error must say so as a failed DOLT_COMMIT does. +func TestEpochWritesMarkAFailedStagedReadIndeterminate(t *testing.T) { + for _, tc := range epochPublishCalls() { + if tc.logsPublishFailure { + continue // a mint logs a failed publication and returns nil + } + t.Run(tc.name, func(t *testing.T) { + f := newEpochFake() + f.stagedReadErr = errors.New("staged-set read failed") + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + err := tc.call(context.Background(), &DoltStore{db: db}) + + requirePublicationFailureReported(t, tc.name, f, err, f.stagedReadErr) + }) + } +} + +// TestEpochWritesReturnWhatAnIndeterminateSQLCommitMayHaveApplied pins what each +// writer hands back when its SQL commit's result is lost (ErrCommitIndeterminate, +// which withRetryTx never retries): the commit may have landed, so a bump +// returns the epoch it would have committed and a mint its address, as a bump +// does when only its Dolt commit fails. The caller then reads CurrentEpoch or +// StillServes to learn which; a zero value would read as nothing applied and +// invite the replay the sentinel forbids. +func TestEpochWritesReturnWhatAnIndeterminateSQLCommitMayHaveApplied(t *testing.T) { + ctx := context.Background() + open := func(t *testing.T) (*DoltStore, *epochFake) { + t.Helper() + f := newEpochFake() + f.connector.commitErr = func() error { return testConnectionLoss } + db := sql.OpenDB(f.connector) + t.Cleanup(func() { _ = db.Close() }) + return &DoltStore{db: db}, f + } + requireIndeterminate := func(t *testing.T, name string, f *epochFake, err error) { + t.Helper() + if !errors.Is(err, ErrCommitIndeterminate) || !errors.Is(err, testConnectionLoss) { + t.Fatalf("%s = %v, want the lost commit response wrapped in ErrCommitIndeterminate", name, err) + } + if events := f.rec.snapshot(); slices.ContainsFunc(events, func(ev string) bool { return strings.Contains(ev, "DOLT_COMMIT") }) { + t.Errorf("%s published a SQL commit whose result was lost; events: %v", name, events) + } + } + + t.Run("BumpEpoch", func(t *testing.T) { + s, f := open(t) + epoch, err := s.BumpEpoch(ctx, epochPublishStoreID, "restore") + requireIndeterminate(t, "BumpEpoch", f, err) + if epoch != 2 { + t.Errorf("BumpEpoch returned epoch %d with an indeterminate commit, want 2, the epoch it may have committed", epoch) + } + }) + t.Run("BumpEpochCarrying", func(t *testing.T) { + s, f := open(t) + epoch, err := s.BumpEpochCarrying(ctx, epochPublishStoreID, "token-scheme-change") + requireIndeterminate(t, "BumpEpochCarrying", f, err) + if epoch != 2 { + t.Errorf("BumpEpochCarrying returned epoch %d with an indeterminate commit, want 2, the epoch it may have committed", epoch) + } + }) + t.Run("MintUnderEpoch", func(t *testing.T) { + s, f := open(t) + address, err := s.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID) + requireIndeterminate(t, "MintUnderEpoch", f, err) + if want := epochPublishAddress(); address != want { + t.Errorf("MintUnderEpoch returned address %q with an indeterminate commit, want %s, the address it may have committed", address, want) + } + }) + t.Run("LoseVersion", func(t *testing.T) { + s, f := open(t) + requireIndeterminate(t, "LoseVersion", f, s.LoseVersion(ctx, epochPublishStoreID, epochPublishAddress())) + }) +} + +// TestEpochWritesReturnNothingWhenTheSQLCommitFailsDefinitely is the other side +// of that rule: a commit error the server answered with (a typed MySQL error, +// which withRetryTx neither retries nor marks indeterminate) means nothing +// landed. Each body ran to the end and computed an epoch or an address, and +// neither may come back with the error. +func TestEpochWritesReturnNothingWhenTheSQLCommitFailsDefinitely(t *testing.T) { + ctx := context.Background() + refused := &mysql.MySQLError{Number: 1105, Message: "commit refused"} + open := func(t *testing.T) *DoltStore { + t.Helper() + f := newEpochFake() + f.connector.commitErr = func() error { return refused } + db := sql.OpenDB(f.connector) + t.Cleanup(func() { _ = db.Close() }) + return &DoltStore{db: db} + } + requireDefinite := func(t *testing.T, name string, err error) { + t.Helper() + if !errors.Is(err, refused) || errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("%s = %v, want the refused commit, not marked ErrCommitIndeterminate", name, err) + } + } + + t.Run("BumpEpoch", func(t *testing.T) { + epoch, err := open(t).BumpEpoch(ctx, epochPublishStoreID, "restore") + requireDefinite(t, "BumpEpoch", err) + if epoch != 0 { + t.Errorf("BumpEpoch returned epoch %d with a refused commit, want 0: nothing applied", epoch) + } + }) + t.Run("BumpEpochCarrying", func(t *testing.T) { + epoch, err := open(t).BumpEpochCarrying(ctx, epochPublishStoreID, "token-scheme-change") + requireDefinite(t, "BumpEpochCarrying", err) + if epoch != 0 { + t.Errorf("BumpEpochCarrying returned epoch %d with a refused commit, want 0: nothing applied", epoch) + } + }) + t.Run("MintUnderEpoch", func(t *testing.T) { + address, err := open(t).MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID) + requireDefinite(t, "MintUnderEpoch", err) + if address != "" { + t.Errorf("MintUnderEpoch returned address %q with a refused commit, want none: nothing applied", address) + } + }) +} + +// TestMintPathsKeepTheirRowsWhenPublicationFails is the other half of that +// rule: swallow-and-log stays acceptable for the frequent, automated mint +// paths, so a failed Dolt commit does not fail a mint whose SQL transaction +// committed (the runIssueOperationTxWithMessage contract: a nil error means +// the data is durable in the working set). +func TestMintPathsKeepTheirRowsWhenPublicationFails(t *testing.T) { + for _, tc := range epochPublishCalls() { + if !tc.logsPublishFailure { + continue // the bumps and the loss surface the failure instead, see above + } + t.Run(tc.name, func(t *testing.T) { + f := newEpochFake() + f.publishErr = errors.New("dolt commit refused") + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + err := tc.call(context.Background(), &DoltStore{db: db}) + + events := f.rec.snapshot() + if !slices.Contains(events, "COMMIT") || slices.Contains(events, "ROLLBACK") { + t.Errorf("%s: a failed Dolt commit must not roll the mint back: want the SQL transaction COMMITTED; events: %v", tc.name, events) + } + if err != nil { + t.Errorf("%s = %v, want nil: the mint's data is durable, so a failed history commit is logged, not returned", tc.name, err) + } + }) + } +} + +// TestCurrentAddressForWritesAndPublishesNothing pins that CurrentAddressFor is a +// read on this leg: it answers from the address's own row, the carry rows and +// the counter, and sends no INSERT, UPDATE or DELETE and no DOLT_ADD or +// DOLT_COMMIT. The read path opens an ordinary transaction and rolls it back, so +// a write it made would vanish without an error; the statements it sends are the +// only place a stray write shows. +func TestCurrentAddressForWritesAndPublishesNothing(t *testing.T) { + f := newEpochFake() + db := sql.OpenDB(f.connector) + defer func() { _ = db.Close() }() + + address := epochPublishAddress() + current, err := (&DoltStore{db: db}).CurrentAddressFor(context.Background(), epochPublishStoreID, address) + if err != nil { + t.Fatalf("CurrentAddressFor: %v", err) + } + if current != address { + t.Errorf("CurrentAddressFor(%s) = %s, want the address itself: the fake serves it as a Live row nothing carried", address, current) + } + + for _, ev := range f.rec.snapshot() { + statement := strings.ToUpper(strings.TrimSpace(ev)) + for _, verb := range []string{"INSERT", "UPDATE", "DELETE", "REPLACE"} { + if strings.HasPrefix(statement, verb) { + t.Errorf("CurrentAddressFor sent %q: a lookup must not write", ev) + } + } + if strings.Contains(statement, "DOLT_ADD") || strings.Contains(statement, "DOLT_COMMIT") { + t.Errorf("CurrentAddressFor sent %q: a lookup has nothing to publish to Dolt history", ev) + } + } +} + +// epochTablesInWorkingSet lists the R20 epoch tables that still appear in +// dolt_status, staged or not: after a write has been published, neither may. +func epochTablesInWorkingSet(status *DoltStatus) []string { + var dirty []string + for _, entry := range append(slices.Clone(status.Staged), status.Unstaged...) { + if entry.Table == "store_epoch" || entry.Table == "epoch_minted_addresses" { + dirty = append(dirty, fmt.Sprintf("%s (%s)", entry.Table, entry.Status)) + } + } + return dirty +} + +// TestFirstMintLeavesTheEpochTablesCommitted reproduces bee-ghosttrack's probe +// for gastownhall/beads#6664 Major 2 on a real server: Status() after the +// first mint on a fresh store read unstaged=[{store_epoch modified}]. +func TestFirstMintLeavesTheEpochTablesCommitted(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + // A fresh branch: store_epoch has no row, so this mint is what seeds it. + if _, err := store.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID); err != nil { + t.Fatalf("MintUnderEpoch on a fresh store: %v", err) + } + + status, err := store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after the first mint the working set still holds %v: the mint's Dolt commit must include the singleton row it seeded", dirty) + } +} + +// TestBumpEpochCarryingLeavesTheEpochTablesCommitted applies the same probe to +// the carrying bump, which writes two tables: the counter, and one carry row per +// live address in epoch_minted_addresses. Publishing only the counter would +// leave those rows modified and uncommitted in the working set. +func TestBumpEpochCarryingLeavesTheEpochTablesCommitted(t *testing.T) { + skipIfNoServer(t) + ctx, cancel := testContext(t) + defer cancel() + store := newMultiConnectionEpochStore(t, ctx, "epoch_carry_published") + + if _, err := store.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID); err != nil { + t.Fatalf("MintUnderEpoch: %v", err) + } + newEpoch, err := store.BumpEpochCarrying(ctx, epochPublishStoreID, "token-scheme-change") + if err != nil { + t.Fatalf("BumpEpochCarrying: %v", err) + } + + // The carry row is what makes the publication more than the counter: without + // it this test would pass for a bump that wrote nothing but store_epoch. + var carryRows int + if err := store.db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM epoch_minted_addresses WHERE minted_id = ? AND minted_epoch = ?", + epochPublishMintedID, newEpoch, + ).Scan(&carryRows); err != nil { + t.Fatalf("count the carry rows: %v", err) + } + if carryRows != 1 { + t.Fatalf("epoch_minted_addresses holds %d rows for %s at epoch %d after the carrying bump, want exactly 1 carry row", carryRows, epochPublishMintedID, newEpoch) + } + + status, err := store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after the carrying bump the working set still holds %v: its Dolt commit must include the carry rows", dirty) + } +} + +// TestLoseVersionLeavesTheEpochTablesCommitted applies the probe to a recorded +// loss, which changes only epoch_minted_addresses: the row it marks must be in +// the Dolt commit, not left modified in the working set. +func TestLoseVersionLeavesTheEpochTablesCommitted(t *testing.T) { + skipIfNoServer(t) + ctx, cancel := testContext(t) + defer cancel() + store := newMultiConnectionEpochStore(t, ctx, "epoch_loss_published") + + address, err := store.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("MintUnderEpoch: %v", err) + } + if err := store.LoseVersion(ctx, epochPublishStoreID, address); err != nil { + t.Fatalf("LoseVersion: %v", err) + } + serves, err := store.StillServes(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("StillServes: %v", err) + } + if serves { + t.Fatalf("StillServes(%s) = true after LoseVersion, want false: the loss must have taken effect for the working-set check below to mean anything", address) + } + + status, err := store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after LoseVersion the working set still holds %v: its Dolt commit must include the row it marked", dirty) + } +} + +// TestCurrentAddressForLeavesAnAbsentStoreEpochRowAbsent is what remains of the +// check that CurrentAddressFor committed a re-seeded singleton row. A store +// whose singleton row is absent is legitimate state — migration 0067 starts the +// table empty and treats "no row" as epoch 1 — and CurrentAddressFor used to +// re-seed it through ensureStoreEpochRow exactly as a first mint does. It only +// reads now: it answers from that default, leaves the singleton absent, and +// leaves nothing in the working set to commit. +func TestCurrentAddressForLeavesAnAbsentStoreEpochRowAbsent(t *testing.T) { + skipIfNoServer(t) + ctx, cancel := testContext(t) + defer cancel() + store := newMultiConnectionEpochStore(t, ctx, "epoch_lookup_absent_row") + + address, err := store.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("MintUnderEpoch: %v", err) + } + // Settle everything so far, then leave the singleton row absent with a + // clean working set. + if _, err := store.CommitAll(ctx, "test: settle the first mint"); err != nil { + t.Fatalf("CommitAll: %v", err) + } + if _, err := store.db.ExecContext(ctx, "DELETE FROM store_epoch"); err != nil { + t.Fatalf("delete the singleton row: %v", err) + } + if _, err := store.CommitAll(ctx, "test: absent singleton row"); err != nil { + t.Fatalf("CommitAll: %v", err) + } + + current, err := store.CurrentAddressFor(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("CurrentAddressFor: %v", err) + } + if current != address { + t.Errorf("CurrentAddressFor(%s) = %s, want the address itself: nothing carried it, and a lookup must not mint a replacement", address, current) + } + + var singletonRows int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM store_epoch").Scan(&singletonRows); err != nil { + t.Fatalf("count the store_epoch rows: %v", err) + } + if singletonRows != 0 { + t.Errorf("store_epoch holds %d rows after CurrentAddressFor, want 0: a lookup must not re-seed the singleton", singletonRows) + } + + status, err := store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after CurrentAddressFor the working set still holds %v: a lookup must leave nothing to commit", dirty) + } +} + +// newMultiConnectionEpochStore opens a store on its own database with several +// connections to one branch, as in production server mode. setupTestStore pins +// a single connection, which cannot overlap two transactions. A store of its own +// also never queues on the shared test database's schema-init lock, whose +// five-second acquire timeout runs out when many parallel tests set up at once +// on a loaded host, so the publication probes that need a database of their own +// for no other reason use it too. +func newMultiConnectionEpochStore(t *testing.T, ctx context.Context, name string) *DoltStore { + t.Helper() + store, err := New(ctx, &Config{ + Path: t.TempDir(), + ServerHost: "127.0.0.1", + ServerPort: testServerPort, + Database: fmt.Sprintf("%s_%d", name, time.Now().UnixNano()), + MaxOpenConns: 4, + CreateIfMissing: true, + CommitterName: "test", + CommitterEmail: "test@example.com", + }) + if err != nil { + t.Fatalf("create store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// TestMintOverlappingABumpLeavesALiveSurvivor measures, on a real server, an +// outcome bee-ghosttrack marked "Unverified" (gastownhall/beads#6664, review +// 5360880888): a mint whose transaction overlaps a BumpEpoch reads the epoch +// before the bump, does not conflict with it (the two write different tables), +// and commits an address minted under the epoch the bump has just replaced. +// +// That address is a Live survivor, and that is the right answer. A restore or +// destructive-reinit bump writes no address rows: it advances only the counter, +// and an address's status is read from its own row, which says only that it was +// minted in an epoch that has happened. So an address minted before the bump is +// still served after it, however the two interleave, exactly as if the two +// calls had run back to back. (Before survival was recorded, this interleaving +// committed an address that already read GoneReorganization, because the bump +// voided every address nobody minted again; that outcome no longer exists.) +// +// The engine does not prevent the overlap, and this test depends on that. +// Measured: SELECT ... FOR UPDATE and LOCK IN SHARE MODE take no lock, and a +// write that changes no value (UPDATE store_epoch SET epoch = epoch matches 0 +// rows) is not a write to the merge. Only a write that CHANGES a cell the bump +// also changes conflicts (Error 1213), and a mint has no cell of its own to +// change. That is also why the token-scheme-change bump, the one bump that reads +// address rows, needs its caller to hold the workspace gate exclusively, as a +// restore does: a mint cannot be made to conflict with it. +// +// If this test fails because the bump timed out or the mint's commit +// conflicted, the engine now prevents the overlap: revisit that requirement and +// this test together. +func TestMintOverlappingABumpLeavesALiveSurvivor(t *testing.T) { + skipIfNoServer(t) + ctx, cancel := testContext(t) + defer cancel() + + store := newMultiConnectionEpochStore(t, ctx, "epoch_overlap") + + // Seed the singleton so the mint below only reads the epoch. + if _, err := store.MintUnderEpoch(ctx, epochPublishStoreID, "seed"); err != nil { + t.Fatalf("seed mint: %v", err) + } + + mintTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin mint transaction: %v", err) + } + defer func() { _ = mintTx.Rollback() }() + address, err := storeops.MintUnderEpochInTx(ctx, mintTx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("mint inside the open transaction: %v", err) + } + + // The bump runs on another connection while the mint transaction is open. + bumpCtx, cancelBump := context.WithTimeout(ctx, 10*time.Second) + defer cancelBump() + bumpedEpoch, err := store.BumpEpoch(bumpCtx, epochPublishStoreID, "restore") + if err != nil { + t.Fatalf("BumpEpoch beside an open mint transaction: %v: the engine now holds the bump back, so the overlap this test documents is prevented", err) + } + if err := mintTx.Commit(); err != nil { + t.Fatalf("mint commit after a concurrent bump: %v: the engine now conflicts the mint, so the overlap this test documents is prevented", err) + } + + serves, err := store.StillServes(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("StillServes: %v", err) + } + if !serves { + t.Errorf("address %s, minted before a bump that committed first, is no longer served: a restore or reinit bump writes no address rows, so the address must survive it", address) + } + answer, err := store.Resolve(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if answer.Restriction != storeops.EpochRestrictionLive { + t.Errorf("Resolve(%s) = %v, want EpochRestrictionLive: the address survived the bump", address, answer.Restriction) + } + if answer.Epoch == nil { + t.Errorf("Resolve(%s).Epoch = nil, want the current epoch %d", address, bumpedEpoch) + } else if *answer.Epoch != bumpedEpoch { + t.Errorf("Resolve(%s).Epoch = %d, want the current epoch %d", address, *answer.Epoch, bumpedEpoch) + } +} + +// TestAMintAfterAConcurrentLossNeverRevivesTheLostAddress choreographs, with no +// timing in it, the interleaving that could bring a lost address back: one +// transaction reads the address Live, another commits the loss of it, and the +// first then mints the same id and commits. +// +// The first transaction read the address before the loss committed, so its +// snapshot holds a Live row and the mint finds nothing to do. A mint never +// writes an existing row (it inserts a root row for a new address and otherwise +// touches nothing), so the first transaction commits no write at all and has +// nothing to conflict with the loss. The loss is the only change to the row, so +// the address ends Gone: no interleaving of a mint and a loss may revive it. +// +// If this test fails because the loss was held back, or because the first +// transaction saw the loss, the engine changed the premise (an open transaction +// reads its own snapshot, and a commit that wrote nothing conflicts with +// nothing) and the interleaving needs another look. +func TestAMintAfterAConcurrentLossNeverRevivesTheLostAddress(t *testing.T) { + skipIfNoServer(t) + ctx, cancel := testContext(t) + defer cancel() + + store := newMultiConnectionEpochStore(t, ctx, "epoch_loss_overlap") + + address, err := store.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("seed mint: %v", err) + } + + // The first transaction reads the address Live and stays open. + mintTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin the first transaction: %v", err) + } + defer func() { _ = mintTx.Rollback() }() + serves, err := storeops.StillServesInTx(ctx, mintTx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("StillServesInTx inside the first transaction: %v", err) + } + if !serves { + t.Fatalf("address %s reads as not served inside the first transaction before any loss: the seed mint did not leave it Live", address) + } + + // The loss commits on another connection while the first transaction is open. + lossCtx, cancelLoss := context.WithTimeout(ctx, 10*time.Second) + defer cancelLoss() + if err := store.LoseVersion(lossCtx, epochPublishStoreID, address); err != nil { + t.Fatalf("LoseVersion beside an open transaction: %v (if the engine now holds the loss back, the interleaving this test choreographs is prevented)", err) + } + + // The first transaction now mints the same id, which is the same address. + minted, err := storeops.MintUnderEpochInTx(ctx, mintTx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("mint inside the first transaction after the loss committed: %v: the transaction should still read the address Live from its own snapshot, and if it now sees the loss this test no longer exercises the interleaving it choreographs", err) + } + if minted != address { + t.Fatalf("mint returned %s, want the same address %s: minting the same id under the same epoch reproduces its address", minted, address) + } + if err := mintTx.Commit(); err != nil { + t.Fatalf("commit of the first transaction: %v: a mint must not write an existing row, so it has nothing to conflict with the loss", err) + } + + serves, err = store.StillServes(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("StillServes: %v", err) + } + if serves { + t.Errorf("address %s is served again after a mint overlapped its loss: no interleaving may revive a lost address", address) + } + answer, err := store.Resolve(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if answer.Restriction != storeops.EpochRestrictionGoneReorganization { + t.Errorf("Resolve(%s) = %v, want EpochRestrictionGoneReorganization: the loss is the only change to the row", address, answer.Restriction) + } +} + +// TestAMintRacingTheMintAndLossOfItsAddressConflictsAtCommit measures what +// upsertEpochMintedAddressInTx says about Dolt. One transaction reads before +// the address exists; another mints the address and loses it, and both commit; +// the first then mints the same id. Its snapshot holds no row at the address, +// so it inserts one, and the INSERT does not fail as a duplicate key: Dolt +// checks the key against the transaction's snapshot. The two inserts conflict at +// COMMIT instead (Error 1213), which withRetryTx answers by running the mint +// again, and run again against the committed row, the mint refuses the address +// (ErrEpochAddressNotServed) rather than reviving a lost one. +// +// If the first transaction's INSERT fails as a duplicate key, the engine now +// checks keys past the snapshot, and the re-read in +// upsertEpochMintedAddressInTx answers this race instead of the commit; if the +// commit succeeds, the engine merged the two rows, and the race needs another +// look. +func TestAMintRacingTheMintAndLossOfItsAddressConflictsAtCommit(t *testing.T) { + skipIfNoServer(t) + ctx, cancel := testContext(t) + defer cancel() + + store := newMultiConnectionEpochStore(t, ctx, "epoch_mint_race") + + // Seed the singleton so the first transaction's mint only reads the epoch. + if _, err := store.MintUnderEpoch(ctx, epochPublishStoreID, "seed"); err != nil { + t.Fatalf("seed mint: %v", err) + } + + // The first transaction reads before the address exists and stays open. + mintTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin the first transaction: %v", err) + } + defer func() { _ = mintTx.Rollback() }() + if _, err := storeops.CurrentEpochInTx(ctx, mintTx, epochPublishStoreID); err != nil { + t.Fatalf("CurrentEpochInTx inside the first transaction: %v", err) + } + + // Another connection mints the address and loses it while the first is open. + otherCtx, cancelOther := context.WithTimeout(ctx, 10*time.Second) + defer cancelOther() + address, err := store.MintUnderEpoch(otherCtx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("MintUnderEpoch beside an open transaction: %v", err) + } + if err := store.LoseVersion(otherCtx, epochPublishStoreID, address); err != nil { + t.Fatalf("LoseVersion beside an open transaction: %v", err) + } + + // The first transaction mints the same id, which is the same address. + minted, err := storeops.MintUnderEpochInTx(ctx, mintTx, epochPublishStoreID, epochPublishMintedID) + if err != nil { + t.Fatalf("mint inside the first transaction: %v: the INSERT now sees a row committed after the transaction's snapshot, so the duplicate-key re-read answers this race, not the commit", err) + } + if minted != address { + t.Fatalf("mint returned %s, want the same address %s: minting the same id under the same epoch reproduces its address", minted, address) + } + err = mintTx.Commit() + if err == nil { + t.Fatalf("the first transaction committed its insert of %s over the committed, lost row: the engine merged the two inserts, and a lost address may come back", address) + } + if !isSerializationError(err) { + t.Fatalf("commit of the first transaction = %v, want a serialization failure (Error 1213), the conflict withRetryTx runs the mint again on", err) + } + + // Run again, as withRetryTx runs it, the mint reads the committed row. + if _, err := store.MintUnderEpoch(ctx, epochPublishStoreID, epochPublishMintedID); !errors.Is(err, storeops.ErrEpochAddressNotServed) { + t.Errorf("mint of %s run again after the conflict = %v, want ErrEpochAddressNotServed: the committed row is lost", epochPublishMintedID, err) + } + serves, err := store.StillServes(ctx, epochPublishStoreID, address) + if err != nil { + t.Fatalf("StillServes: %v", err) + } + if serves { + t.Errorf("address %s is served again after a mint raced its mint and loss: no interleaving may revive a lost address", address) + } +} diff --git a/internal/storage/dolt/expected_revision_contract_test.go b/internal/storage/dolt/expected_revision_contract_test.go index ef76db9b1..1ab82264e 100644 --- a/internal/storage/dolt/expected_revision_contract_test.go +++ b/internal/storage/dolt/expected_revision_contract_test.go @@ -8,11 +8,21 @@ import ( ) // TestExpectedRevisionContract wires this leg into the R16/R17 -// expected-revision contract. Phase 0 leaves every hook nil in every -// backend's fixture kit (architecture §12), so each case below skips by -// name; this file exists so TestEveryLegWiresEveryRoleContract counts this -// leg, and so the cases start running for real the moment this leg's fixture -// kit grows a non-nil hook. +// expected-revision contract. Every hook is nil (architecture §12's Phase 0 +// default), so each case below skips by name; this file exists so +// TestEveryLegWiresEveryRoleContract counts this leg, and so the cases +// start running for real the moment this leg grows a real +// CompareAndSetVersion (gastownhall/beads#6358). +// +// This leg previously wired every hook to an honest "not implemented" stub +// instead of leaving them nil, so each case ran for real and +// failed for that documented reason rather than skipping. That premise — +// that a later slice would give this leg its CompareAndSetVersion — went +// stale once review moved the real implementation to +// gastownhall/beads#6358 instead; a stub whose failure message blames a +// bead that will never land it is worse than an honest skip, so this file +// reverts to nil hooks (gastownhall/beads#6664, bee-ghosttrack review +// 5268699223). func TestExpectedRevisionContract(t *testing.T) { ctx := context.Background() fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "erev"} diff --git a/internal/storage/dolt/retention_epoch_contract_test.go b/internal/storage/dolt/retention_epoch_contract_test.go index ce3569454..83e3a35bc 100644 --- a/internal/storage/dolt/retention_epoch_contract_test.go +++ b/internal/storage/dolt/retention_epoch_contract_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + storeops "github.com/steveyegge/beads/internal/storage/issueops" ) // TestRetentionContract wires this leg into the R20 retention contract. @@ -57,11 +58,20 @@ func TestRetentionContract(t *testing.T) { }) } -// TestEpochContract wires this leg into the R20 epoch contract. Same Phase 0 -// all-nil state as TestRetentionContract above. +// TestEpochContract runs the R20 epoch contract (gastownhall/beads#5898 +// revision 9, #6136) against the server-backed +// store, which reaches internal/storage/issueops's epoch Tx functions +// through this leg's own retrying write transaction or read transaction +// (DoltStore.BumpEpoch/BumpEpochCarrying/MintUnderEpoch/LoseVersion and +// DoltStore.CurrentEpoch/StillServes/Resolve/CurrentAddressFor) — see +// internal/storage/dolt/epoch_cas.go. +// +// All three legs run that one shared body, so this is not an independent +// vote on the design — it is the check on THIS leg's wrapper and this file's +// own conformance-vocabulary translation. func TestEpochContract(t *testing.T) { - ctx := context.Background() - fixture := conformance.EpochFixture{IssuePrefix: "epch"} + fixture, ctx, cleanup := newEpochDoltFixture(t, "epch") + defer cleanup() t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { conformance.RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) @@ -69,4 +79,127 @@ func TestEpochContract(t *testing.T) { t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { conformance.RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) }) + t.Run("TokenSchemeChangeCarriesLiveAddressesAndOnlyThose", func(t *testing.T) { + conformance.RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose(t, ctx, fixture) + }) + t.Run("ACarriedAddressKeepsResolvingAcrossLaterBumps", func(t *testing.T) { + conformance.RunACarriedAddressKeepsResolvingAcrossLaterBumps(t, ctx, fixture) + }) + t.Run("ALostAddressStaysGoneAcrossASchemeChange", func(t *testing.T) { + conformance.RunALostAddressStaysGoneAcrossASchemeChange(t, ctx, fixture) + }) +} + +// newEpochDoltFixture wires this leg's *DoltStore into the R20 epoch +// contract. Like newExpectedRevisionDoltFixture, this fixture needs none of +// roleFixtureKit's CreateIssue/CreateWisp/QueryScalar/CountHistory hooks — +// it has no fields for them — so it does not route through +// newDoltRoleFixtureKit at all; IssuePrefix is set directly, the same value +// the kit would have set verbatim. +func newEpochDoltFixture(t *testing.T, prefix string) (conformance.EpochFixture, context.Context, func()) { + t.Helper() + skipIfNoServer(t) + // A database of its own, not setupTestStore's branch on the shared one. The + // shared database's schema-init lock has a five-second acquire timeout, and a + // setup that queues behind another parallel test's runs out of it under load. + // The context is built after the open, so the open cannot drain its deadline. + openCtx, cancelOpen := testContext(t) + store := newMultiConnectionEpochStore(t, openCtx, "epoch_contract") + cancelOpen() + ctx, cancel := testContext(t) + return conformance.EpochFixture{ + IssuePrefix: prefix, + CurrentEpoch: epochDoltCurrentEpoch(store), + BumpEpoch: epochDoltBumpEpoch(store), + MintUnderEpoch: epochDoltMintUnderEpoch(store), + StillServes: epochDoltStillServes(store), + Resolve: epochDoltResolve(store), + CurrentAddressFor: epochDoltCurrentAddressFor(store), + LoseVersion: epochDoltLoseVersion(store), + }, ctx, cancel +} + +func epochDoltCurrentEpoch(store *DoltStore) func(ctx context.Context, storeID string) (int, error) { + return func(ctx context.Context, storeID string) (int, error) { + return store.CurrentEpoch(ctx, storeID) + } +} + +// epochDoltBumpEpoch dispatches on the typed trigger: a token-scheme change is +// the one bump that also carries every live address to the new epoch, and a +// restore or destructive reinit advances only the counter. The dispatch lives +// in this file, the translation boundary, because the shared body has no +// switch on the reason string. +func epochDoltBumpEpoch(store *DoltStore) func(ctx context.Context, storeID string, trigger conformance.EpochBumpTrigger) (int, error) { + return func(ctx context.Context, storeID string, trigger conformance.EpochBumpTrigger) (int, error) { + if trigger == conformance.EpochBumpTriggerTokenSchemeChange { + return store.BumpEpochCarrying(ctx, storeID, trigger.String()) + } + return store.BumpEpoch(ctx, storeID, trigger.String()) + } +} + +func epochDoltMintUnderEpoch(store *DoltStore) func(ctx context.Context, storeID, id string) (conformance.Address, error) { + return func(ctx context.Context, storeID, id string) (conformance.Address, error) { + address, err := store.MintUnderEpoch(ctx, storeID, id) + if err != nil { + return "", err + } + return conformance.Address(address), nil + } +} + +func epochDoltStillServes(store *DoltStore) func(ctx context.Context, storeID string, address conformance.Address) (bool, error) { + return func(ctx context.Context, storeID string, address conformance.Address) (bool, error) { + return store.StillServes(ctx, storeID, string(address)) + } +} + +func epochDoltResolve(store *DoltStore) func(ctx context.Context, storeID string, address conformance.Address) (conformance.RetentionAnswer, error) { + return func(ctx context.Context, storeID string, address conformance.Address) (conformance.RetentionAnswer, error) { + result, err := store.Resolve(ctx, storeID, string(address)) + if err != nil { + return conformance.RetentionAnswer{}, err + } + return epochResolveResultToConformance(result), nil + } +} + +func epochDoltCurrentAddressFor(store *DoltStore) func(ctx context.Context, storeID string, oldAddress conformance.Address) (conformance.Address, error) { + return func(ctx context.Context, storeID string, oldAddress conformance.Address) (conformance.Address, error) { + address, err := store.CurrentAddressFor(ctx, storeID, string(oldAddress)) + if err != nil { + return "", err + } + return conformance.Address(address), nil + } +} + +func epochDoltLoseVersion(store *DoltStore) func(ctx context.Context, storeID string, address conformance.Address) error { + return func(ctx context.Context, storeID string, address conformance.Address) error { + return store.LoseVersion(ctx, storeID, string(address)) + } +} + +// epochRestrictionToConformance translates this leg's local +// storeops.EpochRestriction into backend/conformance's own Restriction +// vocabulary — the contract-test file is the translation boundary, per +// epoch_cas.go's package doc. +func epochRestrictionToConformance(r storeops.EpochRestriction) conformance.Restriction { + switch r { + case storeops.EpochRestrictionLive: + return conformance.RestrictionLive + case storeops.EpochRestrictionGoneReorganization: + return conformance.RestrictionGoneReorganization + default: + return conformance.RestrictionUnknown + } +} + +func epochResolveResultToConformance(r storeops.EpochResolveResult) conformance.RetentionAnswer { + return conformance.RetentionAnswer{ + Restriction: epochRestrictionToConformance(r.Restriction), + ProducingStore: r.ProducingStore, + Epoch: r.Epoch, + } } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index c7744f5a8..0391b56e9 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -26,6 +26,7 @@ go_library( "dependency_editor.go", "detection.go", "edgereader.go", + "epoch_cas.go", "federation.go", "get_issue.go", "graph_counter.go", @@ -150,6 +151,8 @@ go_test( "dualwrite_history_contract_test.go", "dualwrite_participation_generation_contract_test.go", "edge_reader_contract_test.go", + "epoch_cas_publish_internal_test.go", + "epoch_cas_status_test.go", "events_since_test.go", "expected_revision_contract_test.go", "external_dependencies_test.go", diff --git a/internal/storage/embeddeddolt/epoch_cas.go b/internal/storage/embeddeddolt/epoch_cas.go new file mode 100644 index 000000000..abc551087 --- /dev/null +++ b/internal/storage/embeddeddolt/epoch_cas.go @@ -0,0 +1,197 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" +) + +// CurrentEpoch, BumpEpoch, BumpEpochCarrying, MintUnderEpoch, LoseVersion, +// StillServes, Resolve and CurrentAddressFor give this leg R20's +// epoch-transition enforcement +// (gastownhall/beads#5898 revision 9, #6136), +// backed by store_epoch (migration 0067) and epoch_minted_addresses +// (migration 0072). See internal/storage/dolt/epoch_cas.go's matching +// header comment for why these are direct methods on *EmbeddedDoltStore +// rather than a wrapper struct or root-storage-package types. +// +// The writers publish after their SQL transaction commits, and a failed Dolt +// commit is returned (runTransactionWithMessage), so all four meet the +// ordering the server leg is held to. They also return what the server leg +// returns once the SQL commit has landed or may have: runTransactionWithMessage +// wraps every such error in storage.ErrCommitIndeterminate, and a bump hands +// back the epoch its body set (failedBump) and a mint its address, never the +// zero value that would read as nothing applied. Unlike the server leg's mint, +// which logs a failed Dolt commit, this mint returns it, as every write on this +// leg does. A mint must still stage store_epoch (storeops.EpochMintDirtyTables): +// its first use inserts the singleton row. + +// CurrentEpoch reports storeID's current epoch generation. +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +func (s *EmbeddedDoltStore) CurrentEpoch(ctx context.Context, storeID string) (int, error) { + var epoch int + if err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + epoch, err = storeops.CurrentEpochInTx(ctx, tx, storeID) + return err + }); err != nil { + return 0, err + } + return epoch, nil +} + +// BumpEpoch advances storeID's epoch generation by one for reason. An error +// wrapping storage.ErrCommitIndeterminate comes back with the epoch the bump +// committed, or may have: the caller must not bump again to retry. +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +func (s *EmbeddedDoltStore) BumpEpoch(ctx context.Context, storeID, reason string) (int, error) { + var epoch int + if err := s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + epoch, err = storeops.BumpEpochInTx(ctx, tx, storeID, reason) + if err != nil { + return nil, "", err + } + return storeops.ChangedTables{"store_epoch": true}, + fmt.Sprintf("bd: bump epoch for %s (%s)", storeID, reason), nil + }); err != nil { + return failedBump(storeID, epoch, err) + } + return epoch, nil +} + +// failedBump is what a bump returns when it fails. An error wrapping +// storage.ErrCommitIndeterminate comes from a body that ran to completion and a +// SQL commit that landed or may have (runTransactionWithMessage), so the epoch +// that body set is returned with it, and the caller reads CurrentEpoch to learn +// which. Any other error means nothing landed. +func failedBump(storeID string, epoch int, err error) (int, error) { + if errors.Is(err, storage.ErrCommitIndeterminate) { + return epoch, fmt.Errorf("epoch bump for %s may have applied (epoch %d): %w", storeID, epoch, err) + } + return 0, err +} + +// BumpEpochCarrying is the token-scheme-change bump: BumpEpoch, and also the +// carry of every live address in the database to the new epoch +// (storeops.BumpEpochCarryingInTx). It writes store_epoch and +// epoch_minted_addresses, so it publishes both. +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +// +// It fails as BumpEpoch does (failedBump). +// +// CALLERS MUST HOLD THE WORKSPACE GATE (internal/workspacegate) EXCLUSIVELY FOR +// THE BUMP, as a restore does: see storeops.BumpEpochCarryingInTx. +func (s *EmbeddedDoltStore) BumpEpochCarrying(ctx context.Context, storeID, reason string) (int, error) { + var epoch int + if err := s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + epoch, err = storeops.BumpEpochCarryingInTx(ctx, tx, storeID, reason) + if err != nil { + return nil, "", err + } + return storeops.ChangedTables{"store_epoch": true, "epoch_minted_addresses": true}, + fmt.Sprintf("bd: carry addresses to epoch %d for %s (%s)", epoch, storeID, reason), nil + }); err != nil { + return failedBump(storeID, epoch, err) + } + return epoch, nil +} + +// MintUnderEpoch mints id's address under storeID's current epoch. An error +// wrapping storage.ErrCommitIndeterminate comes back with the address the mint +// committed, or may have; StillServes says whether it did. +// +// Embedded transactions serialize (see recheckBlockedAfterCommit), so a mint +// cannot overlap a BumpEpoch here the way it can on the server leg (see +// TestMintOverlappingABumpLeavesALiveSurvivor in the dolt package). +func (s *EmbeddedDoltStore) MintUnderEpoch(ctx context.Context, storeID, id string) (string, error) { + var address string + if err := s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + address, err = storeops.MintUnderEpochInTx(ctx, tx, storeID, id) + if err != nil { + return nil, "", err + } + return storeops.EpochMintDirtyTables(), + fmt.Sprintf("bd: mint %s under epoch for %s", id, storeID), nil + }); err != nil { + if errors.Is(err, storage.ErrCommitIndeterminate) { + return address, err + } + return "", err + } + return address, nil +} + +// StillServes reports whether address is still served under storeID's +// current epoch. +func (s *EmbeddedDoltStore) StillServes(ctx context.Context, storeID, address string) (bool, error) { + var serves bool + if err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + serves, err = storeops.StillServesInTx(ctx, tx, storeID, address) + return err + }); err != nil { + return false, err + } + return serves, nil +} + +// Resolve answers R20's epoch-only restriction for address. +func (s *EmbeddedDoltStore) Resolve(ctx context.Context, storeID, address string) (storeops.EpochResolveResult, error) { + var result storeops.EpochResolveResult + if err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = storeops.ResolveEpochInTx(ctx, tx, storeID, address) + return err + }); err != nil { + return storeops.EpochResolveResult{}, err + } + return result, nil +} + +// CurrentAddressFor reports the address oldAddress now resolves to: the newest +// address of its lineage, or oldAddress itself when nothing carried it. It only +// reads, so it runs on a read-only connection and publishes nothing. +func (s *EmbeddedDoltStore) CurrentAddressFor(ctx context.Context, storeID, oldAddress string) (string, error) { + var address string + if err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + address, err = storeops.CurrentAddressForInTx(ctx, tx, storeID, oldAddress) + return err + }); err != nil { + return "", err + } + return address, nil +} + +// LoseVersion records that storeID no longer serves the Version named by +// address (storeops.LoseVersionInTx). It writes epoch_minted_addresses, so it +// publishes that table, and a failed Dolt commit is returned, wrapping +// storage.ErrCommitIndeterminate: a loss its caller believes recorded must not +// be left unpublished. +func (s *EmbeddedDoltStore) LoseVersion(ctx context.Context, storeID, address string) error { + return s.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + if err := storeops.LoseVersionInTx(ctx, tx, storeID, address); err != nil { + return nil, "", err + } + return storeops.ChangedTables{"epoch_minted_addresses": true}, + fmt.Sprintf("bd: lose version %s for %s", address, storeID), nil + }) +} diff --git a/internal/storage/embeddeddolt/epoch_cas_publish_internal_test.go b/internal/storage/embeddeddolt/epoch_cas_publish_internal_test.go new file mode 100644 index 000000000..96623da36 --- /dev/null +++ b/internal/storage/embeddeddolt/epoch_cas_publish_internal_test.go @@ -0,0 +1,141 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/storage" +) + +// These tests are this leg's half of the publication-failure rule for the R20 +// epoch writers (gastownhall/beads#6664); the server leg's half is +// internal/storage/dolt/epoch_cas_publish_test.go. A write whose SQL commit has +// landed and whose Dolt commit then fails must wrap +// storage.ErrCommitIndeterminate and hand back what it applied, so that nothing +// replays a bump that has already advanced the counter. + +// withFailedPublication runs write with every Dolt commit after its SQL commit +// refused: afterSQLCommit marks store read-only in that window, so the rows have +// landed when withMutatingDBConn refuses to stage and commit them. +func withFailedPublication(t *testing.T, store *EmbeddedDoltStore, write func()) { + t.Helper() + saved := afterSQLCommit + afterSQLCommit = func() { store.readOnly = true } + defer func() { + afterSQLCommit = saved + store.readOnly = false + }() + write() +} + +func TestEpochWritesReportAnUnpublishedCommitAsIndeterminate(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "epochpub", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + const storeID = "epochpub-store" + + requireUnpublished := func(t *testing.T, name string, err error) { + t.Helper() + if !errors.Is(err, ErrReadOnly) { + t.Fatalf("%s = %v, want the refused Dolt commit (ErrReadOnly)", name, err) + } + if !errors.Is(err, storage.ErrCommitIndeterminate) { + t.Fatalf("%s = %v, want it to wrap storage.ErrCommitIndeterminate: the write has applied, and a caller that takes the error for a write that did not would replay it", name, err) + } + } + currentEpoch := func(t *testing.T) int { + t.Helper() + epoch, err := store.CurrentEpoch(ctx, storeID) + if err != nil { + t.Fatalf("CurrentEpoch: %v", err) + } + return epoch + } + stillServes := func(t *testing.T, address string) bool { + t.Helper() + serves, err := store.StillServes(ctx, storeID, address) + if err != nil { + t.Fatalf("StillServes(%s): %v", address, err) + } + return serves + } + + for _, tc := range []struct { + name string + bump func(ctx context.Context, storeID, reason string) (int, error) + }{ + {name: "BumpEpoch", bump: store.BumpEpoch}, + {name: "BumpEpochCarrying", bump: store.BumpEpochCarrying}, + } { + t.Run(tc.name, func(t *testing.T) { + before := currentEpoch(t) + var epoch int + var err error + withFailedPublication(t, store, func() { + epoch, err = tc.bump(ctx, storeID, "publication fails") + }) + requireUnpublished(t, tc.name, err) + if epoch != before+1 { + t.Errorf("%s returned epoch %d with the publication error, want %d: the bump applied, the error only says its Dolt commit did not", tc.name, epoch, before+1) + } + if got := currentEpoch(t); got != before+1 { + t.Errorf("CurrentEpoch after an unpublished %s = %d, want %d", tc.name, got, before+1) + } + }) + } + + t.Run("MintUnderEpoch", func(t *testing.T) { + var address string + var err error + withFailedPublication(t, store, func() { + address, err = store.MintUnderEpoch(ctx, storeID, "record-a") + }) + requireUnpublished(t, "MintUnderEpoch", err) + if address == "" { + t.Fatal("MintUnderEpoch returned no address with the publication error, want the address it minted") + } + if !stillServes(t, address) { + t.Errorf("StillServes(%s) = false after an unpublished mint, want the minted address served", address) + } + }) + + t.Run("LoseVersion", func(t *testing.T) { + address, err := store.MintUnderEpoch(ctx, storeID, "record-b") + if err != nil { + t.Fatalf("MintUnderEpoch: %v", err) + } + withFailedPublication(t, store, func() { + err = store.LoseVersion(ctx, storeID, address) + }) + requireUnpublished(t, "LoseVersion", err) + if stillServes(t, address) { + t.Errorf("StillServes(%s) = true after an unpublished loss, want the loss applied", address) + } + }) + + // A write refused before its SQL commit applied nothing, so its error must + // not claim it may have. + t.Run("a bump refused before its SQL commit", func(t *testing.T) { + before := currentEpoch(t) + store.readOnly = true + epoch, err := store.BumpEpoch(ctx, storeID, "refused") + store.readOnly = false + if !errors.Is(err, ErrReadOnly) || errors.Is(err, storage.ErrCommitIndeterminate) { + t.Fatalf("BumpEpoch on a read-only store = %v, want ErrReadOnly, not marked storage.ErrCommitIndeterminate", err) + } + if epoch != 0 { + t.Errorf("BumpEpoch on a read-only store returned epoch %d, want 0", epoch) + } + if got := currentEpoch(t); got != before { + t.Errorf("CurrentEpoch after a refused bump = %d, want %d", got, before) + } + }) +} diff --git a/internal/storage/embeddeddolt/epoch_cas_status_test.go b/internal/storage/embeddeddolt/epoch_cas_status_test.go new file mode 100644 index 000000000..ac28e12a5 --- /dev/null +++ b/internal/storage/embeddeddolt/epoch_cas_status_test.go @@ -0,0 +1,562 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "fmt" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" +) + +// These tests are the embedded-leg half of gastownhall/beads#6664 Major 2 +// (bee-ghosttrack, review 5360880888). +// MintUnderEpoch and CurrentAddressFor staged only epoch_minted_addresses, +// while ensureStoreEpochRow lazily inserts store_epoch's singleton row on +// first use, so that row stayed modified and uncommitted. The embedded leg +// never published inside the transaction as the server leg did; it is not free +// of this omission. +// +// The tests after those run the rest of the epoch write surface on the real +// engine: the token-scheme-change carry (BumpEpochCarrying), the explicit loss +// (LoseVersion), and CurrentAddressFor, which only reads. They check what the +// statement-level unit tests cannot: set semantics across rows, a bump that +// rolls back, and what the working set holds once each write is published. + +// epochTablesInWorkingSet lists the R20 epoch tables that still appear in +// dolt_status, staged or not: after a write has been published, neither may. +func epochTablesInWorkingSet(status *storage.Status) []string { + var dirty []string + for _, entry := range append(slices.Clone(status.Staged), status.Unstaged...) { + if entry.Table == "store_epoch" || entry.Table == "epoch_minted_addresses" { + dirty = append(dirty, fmt.Sprintf("%s (%s)", entry.Table, entry.Status)) + } + } + return dirty +} + +// TestFirstMintLeavesTheEpochTablesCommitted reproduces bee-ghosttrack's probe +// (Status() after the first mint on a fresh store read +// unstaged=[{store_epoch modified}]) on the embedded leg. +func TestFirstMintLeavesTheEpochTablesCommitted(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + + // A pristine store: store_epoch has no row, so this mint is what seeds it. + if _, err := te.store.MintUnderEpoch(ctx, "epoch-store", "record-a"); err != nil { + t.Fatalf("MintUnderEpoch on a fresh store: %v", err) + } + + status, err := te.store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after the first mint the working set still holds %v: the mint's Dolt commit must include the singleton row it seeded", dirty) + } +} + +// TestCurrentAddressForNeverReseedsAnAbsentStoreEpochRow is the successor of the +// test that pinned CurrentAddressFor committing a store_epoch row it re-seeded. +// CurrentAddressFor no longer writes anything (gastownhall/beads#6664, +// bee-ghosttrack review 5360880888, Major 1: a lookup must not mint), so the +// same setup now pins the opposite. A store whose singleton row is absent is +// legitimate state — migration 0067 starts the table empty and treats "no row" +// as epoch 1 — and CurrentAddressFor answers from that default: it returns the +// address it was given, leaves the working set clean, and does not create the +// row. +func TestCurrentAddressForNeverReseedsAnAbsentStoreEpochRow(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + + address, err := te.store.MintUnderEpoch(ctx, "epoch-store", "record-a") + if err != nil { + t.Fatalf("MintUnderEpoch: %v", err) + } + // Settle everything so far, then leave the singleton row absent with a + // clean working set. + if _, err := te.store.CommitAll(ctx, "test: settle the first mint"); err != nil { + t.Fatalf("CommitAll: %v", err) + } + te.exec(t, ctx, "DELETE FROM store_epoch") + if _, err := te.store.CommitAll(ctx, "test: absent singleton row"); err != nil { + t.Fatalf("CommitAll: %v", err) + } + + got, err := te.store.CurrentAddressFor(ctx, "epoch-store", address) + if err != nil { + t.Fatalf("CurrentAddressFor: %v", err) + } + if got != address { + t.Errorf("CurrentAddressFor = %s, want the address it was given (%s): nothing carried the Version, so it resolves to itself", got, address) + } + + var rows int + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM store_epoch", nil, &rows) + if rows != 0 { + t.Errorf("store_epoch holds %d rows after CurrentAddressFor, want 0: a lookup must not re-create the singleton row", rows) + } + + status, err := te.store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after CurrentAddressFor the working set holds %v: a lookup changes nothing, so there is nothing to commit", dirty) + } +} + +// TestBumpEpochCarryingLeavesTheEpochTablesCommitted is the publication rule +// for the token-scheme-change bump. It writes store_epoch (the counter) and +// epoch_minted_addresses (the carry rows), so its Dolt commit has to stage +// both: a bump that staged only store_epoch would leave the carry rows +// modified and uncommitted. +func TestBumpEpochCarryingLeavesTheEpochTablesCommitted(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + epochMint(t, ctx, te, store, "record-a") + newEpoch := epochCarry(t, ctx, te, store) + + // The carry has to have happened for the check below to mean anything. + if got := epochRowsAt(t, ctx, te, store, newEpoch); got != 1 { + t.Fatalf("after the carrying bump to epoch %d the store holds %d rows minted at that epoch, want 1 carry row", newEpoch, got) + } + status, err := te.store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after the carrying bump the working set still holds %v: its Dolt commit must include the carry rows as well as the counter", dirty) + } +} + +// TestLoseVersionLeavesTheEpochTablesCommitted is the same rule for an explicit +// loss, which writes epoch_minted_addresses alone: a loss the caller believes +// recorded must not be left modified and uncommitted. +func TestLoseVersionLeavesTheEpochTablesCommitted(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + address := epochMint(t, ctx, te, store, "record-a") + if err := te.store.LoseVersion(ctx, store, address); err != nil { + t.Fatalf("LoseVersion: %v", err) + } + + // The loss has to have been recorded for the check below to mean anything. + epoch, err := te.store.CurrentEpoch(ctx, store) + if err != nil { + t.Fatalf("CurrentEpoch: %v", err) + } + epochRequireGone(t, ctx, te, store, address, epoch, "LoseVersion recorded the loss of this address") + + status, err := te.store.Status(ctx) + if err != nil { + t.Fatalf("Status: %v", err) + } + if dirty := epochTablesInWorkingSet(status); len(dirty) > 0 { + t.Errorf("after LoseVersion the working set still holds %v: the loss must be published with the row it changed", dirty) + } +} + +// TestBumpEpochCarryingWritesOneCarryRowPerLiveIDNamingTheLineageRoot pins the +// write surface of the token-scheme-change bump on the real engine +// (gastownhall/beads#6664, bee-ghosttrack review 5360880888, Major 1): the bump +// records the carry itself, one row per live id at the new epoch, instead of +// leaving it to be inferred from a later mint. Each carry row names the lineage +// ROOT in carried_from, so a second carry still names the first root, lineages +// stay flat, and every address of a lineage is one lookup from the newest. +func TestBumpEpochCarryingWritesOneCarryRowPerLiveIDNamingTheLineageRoot(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + rootA := epochMint(t, ctx, te, store, "record-a") + rootB := epochMint(t, ctx, te, store, "record-b") + + firstEpoch := epochCarry(t, ctx, te, store) + if got := epochRowsAt(t, ctx, te, store, firstEpoch); got != 2 { + t.Fatalf("after the carrying bump to epoch %d the store holds %d rows minted at that epoch, want 2: one carry row per live id", firstEpoch, got) + } + carryA := epochRowAddress(t, ctx, te, store, "record-a", firstEpoch) + carryB := epochRowAddress(t, ctx, te, store, "record-b", firstEpoch) + for _, c := range []struct{ id, carry, root string }{ + {"record-a", carryA, rootA}, + {"record-b", carryB, rootB}, + } { + if carriedFrom := epochCarriedFrom(t, ctx, te, c.carry); !carriedFrom.Valid || carriedFrom.String != c.root { + t.Errorf("the carry row of %s names carried_from %v, want its root %s", c.id, carriedFrom, c.root) + } + if goneAt := epochGoneAt(t, ctx, te, c.carry); goneAt.Valid { + t.Errorf("the carry row of %s is already marked gone at epoch %d, want it live", c.id, goneAt.Int64) + } + } + for _, c := range []struct{ old, carry string }{{rootA, carryA}, {rootB, carryB}} { + if got, err := te.store.CurrentAddressFor(ctx, store, c.old); err != nil || got != c.carry { + t.Errorf("CurrentAddressFor(%s) = %q, %v, want the carry address %s", c.old, got, err, c.carry) + } + epochRequireServed(t, ctx, te, store, c.old, "the old address keeps resolving through the carry") + epochRequireServed(t, ctx, te, store, c.carry, "the carry address is served under the new epoch") + } + + // A second carry still names the first ROOT, not the previous carry row. + secondEpoch := epochCarry(t, ctx, te, store) + secondCarryA := epochRowAddress(t, ctx, te, store, "record-a", secondEpoch) + if carriedFrom := epochCarriedFrom(t, ctx, te, secondCarryA); !carriedFrom.Valid || carriedFrom.String != rootA { + t.Errorf("the second carry row of record-a names carried_from %v, want the first root %s: carried_from names the root, never the previous carry", carriedFrom, rootA) + } + for _, member := range []string{rootA, carryA} { + if got, err := te.store.CurrentAddressFor(ctx, store, member); err != nil || got != secondCarryA { + t.Errorf("CurrentAddressFor(%s) = %q, %v, want %s: every address of the lineage points straight at the newest one", member, got, err, secondCarryA) + } + } +} + +// TestBumpEpochCarryingCarriesOnlyTheLineageHoldingTheNewestRow pins the +// collision rule. A mint never retires an address, so one id can have two live +// lineages: here a second root minted after a restore. Both cannot receive an +// address, because an address is a function of the store, the id and the epoch +// and so is one per id per epoch. The lineage holding the newest row is +// carried, and the other is marked gone at the bump's epoch: over-voiding is +// the safe direction, reviving is not. +func TestBumpEpochCarryingCarriesOnlyTheLineageHoldingTheNewestRow(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + first := epochMint(t, ctx, te, store, "record-x") + epochBump(t, ctx, te, store) + second := epochMint(t, ctx, te, store, "record-x") + if first == second { + t.Fatalf("minting record-x again after a bump returned %s, want a second root for the new epoch", first) + } + + newEpoch := epochCarry(t, ctx, te, store) + + carry := epochRowAddress(t, ctx, te, store, "record-x", newEpoch) + if carriedFrom := epochCarriedFrom(t, ctx, te, carry); !carriedFrom.Valid || carriedFrom.String != second { + t.Errorf("the carry row names carried_from %v, want %s: the lineage holding the newest row is the one carried", carriedFrom, second) + } + if goneAt := epochGoneAt(t, ctx, te, first); !goneAt.Valid || goneAt.Int64 != int64(newEpoch) { + t.Errorf("the losing lineage's root has gone_at_epoch %v, want %d: it is marked gone at the epoch of the bump", goneAt, newEpoch) + } + if rows := epochCarryRowsOf(t, ctx, te, first); rows != 0 { + t.Errorf("%d carry rows name the losing lineage's root, want 0: a lineage that lost the collision is not carried", rows) + } + + epochRequireGone(t, ctx, te, store, first, newEpoch, "the lineage that lost the collision is no longer served") + epochRequireServed(t, ctx, te, store, second, "the lineage holding the newest row is still served") + if got, err := te.store.CurrentAddressFor(ctx, store, second); err != nil || got != carry { + t.Errorf("CurrentAddressFor(%s) = %q, %v, want the carry address %s", second, got, err, carry) + } + if got, err := te.store.CurrentAddressFor(ctx, store, first); err == nil { + t.Errorf("CurrentAddressFor(the losing lineage's root) = %s with no error, want an error: a lost lineage has no current address", got) + } +} + +// TestBumpEpochCarryingNeitherCarriesNorServesARowFromAFutureEpoch pins how the +// carry treats an anomaly a rewound counter can leave behind: a row minted in +// an epoch that has not happened yet. It is not carried, and it is not served +// either: a status read fails closed rather than trusting it. The carried live +// row beside it is what makes the check mean anything. +func TestBumpEpochCarryingNeitherCarriesNorServesARowFromAFutureEpoch(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + live := epochMint(t, ctx, te, store, "record-a") + future := epochMint(t, ctx, te, store, "record-future") + te.exec(t, ctx, "UPDATE epoch_minted_addresses SET minted_epoch = minted_epoch + 5 WHERE address = ?", future) + + newEpoch := epochCarry(t, ctx, te, store) + + if got := epochRowsAt(t, ctx, te, store, newEpoch); got != 1 { + t.Fatalf("after the carrying bump to epoch %d the store holds %d rows minted at that epoch, want 1: the live row is carried and the row from a future epoch is not", newEpoch, got) + } + carry := epochRowAddress(t, ctx, te, store, "record-a", newEpoch) + if carriedFrom := epochCarriedFrom(t, ctx, te, carry); !carriedFrom.Valid || carriedFrom.String != live { + t.Errorf("the carry row names carried_from %v, want %s", carriedFrom, live) + } + if rows := epochCarryRowsOf(t, ctx, te, future); rows != 0 { + t.Errorf("%d carry rows name the future-epoch row, want 0: a row from an epoch that has not happened is not carried", rows) + } + epochRequireServed(t, ctx, te, store, live, "the live row is served") + epochRequireGone(t, ctx, te, store, future, newEpoch, "a row minted in an epoch that has not happened yet fails closed") +} + +// TestBumpEpochCarryingFailsAndRollsBackWhenARowAlreadyExistsAtTheNewEpoch pins +// the one place a duplicate key always fails the write. A mint whose insert +// collides with a row reads it and keeps the address when the row is served, +// but a row already sitting at the carry's address means the epoch counter +// rewound. The bump must then fail and roll back whole, including its increment +// of the counter, rather than report a carry it did not make. +func TestBumpEpochCarryingFailsAndRollsBackWhenARowAlreadyExistsAtTheNewEpoch(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + epochMint(t, ctx, te, store, "record-a") + startEpoch, err := te.store.CurrentEpoch(ctx, store) + if err != nil { + t.Fatalf("CurrentEpoch: %v", err) + } + nextEpoch := epochBump(t, ctx, te, store) + epochMint(t, ctx, te, store, "record-a") // a row now exists at nextEpoch + te.exec(t, ctx, "UPDATE store_epoch SET epoch = ? WHERE id = 1", startEpoch) + + if got, err := te.store.BumpEpochCarrying(ctx, store, "token-scheme-change"); err == nil { + t.Errorf("BumpEpochCarrying = epoch %d with no error although a row already exists at epoch %d: the counter rewound, so the carry must fail the bump", got, nextEpoch) + } + epoch, err := te.store.CurrentEpoch(ctx, store) + if err != nil { + t.Fatalf("CurrentEpoch: %v", err) + } + if epoch != startEpoch { + t.Errorf("CurrentEpoch = %d after the failed bump, want %d: a failed carry rolls the counter's increment back with it", epoch, startEpoch) + } +} + +// TestBumpEpochCarryingCarriesTheLiveRowsOfEveryStoreInTheDatabase pins why the +// carry is database-wide. The epoch counter is per database, so one bump moves +// the epoch seen through every storeID; carrying only the rows of the storeID +// that asked would strand every other store's addresses at the scheme boundary. +func TestBumpEpochCarryingCarriesTheLiveRowsOfEveryStoreInTheDatabase(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const first, second = "epoch-store-one", "epoch-store-two" + + oldFirst := epochMint(t, ctx, te, first, "record-a") + oldSecond := epochMint(t, ctx, te, second, "record-b") + + newEpoch := epochCarry(t, ctx, te, first) // made under the first storeID only + + for _, c := range []struct{ store, id, old string }{ + {first, "record-a", oldFirst}, + {second, "record-b", oldSecond}, + } { + carry := epochRowAddress(t, ctx, te, c.store, c.id, newEpoch) + if carriedFrom := epochCarriedFrom(t, ctx, te, carry); !carriedFrom.Valid || carriedFrom.String != c.old { + t.Errorf("the carry row of %s in %s names carried_from %v, want its root %s", c.id, c.store, carriedFrom, c.old) + } + if got, err := te.store.CurrentAddressFor(ctx, c.store, c.old); err != nil || got != carry { + t.Errorf("CurrentAddressFor(%s) in %s = %q, %v, want the carry address %s", c.old, c.store, got, err, carry) + } + epochRequireServed(t, ctx, te, c.store, c.old, "the old address keeps resolving through the carry") + epochRequireServed(t, ctx, te, c.store, carry, "the carry address is served under the new epoch") + } +} + +// TestBumpEpochCarryingDoesNotCarryALostLineage pins that loss is terminal for +// the carry: a lineage the store lost before the bump gets no carry row and +// stays Gone, while the live lineage beside it is carried. +func TestBumpEpochCarryingDoesNotCarryALostLineage(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const store = "epoch-store" + + kept := epochMint(t, ctx, te, store, "record-kept") + lost := epochMint(t, ctx, te, store, "record-lost") + if err := te.store.LoseVersion(ctx, store, lost); err != nil { + t.Fatalf("LoseVersion(%s): %v", lost, err) + } + + newEpoch := epochCarry(t, ctx, te, store) + + if got := epochRowsAt(t, ctx, te, store, newEpoch); got != 1 { + t.Fatalf("after the carrying bump to epoch %d the store holds %d rows minted at that epoch, want 1: only the live lineage is carried", newEpoch, got) + } + epochRowAddress(t, ctx, te, store, "record-kept", newEpoch) + if rows := epochCarryRowsOf(t, ctx, te, lost); rows != 0 { + t.Errorf("%d carry rows name the lost lineage's root, want 0: a lost lineage is not carried", rows) + } + epochRequireGone(t, ctx, te, store, lost, newEpoch, "the carry must not bring a lost lineage back") + if got, err := te.store.CurrentAddressFor(ctx, store, lost); err == nil { + t.Errorf("CurrentAddressFor(a lost address) = %s with no error, want an error", got) + } + epochRequireServed(t, ctx, te, store, kept, "the live lineage is served") +} + +// TestABumpUnderOneStoreIDMovesTheEpochSeenThroughEveryStoreID pins the fact the +// bump documentation states: the epoch counter is per database, not per store, +// and storeID appears only in error text. It deliberately pins no guard in the +// shared body, because the conformance suite uses several synthetic storeIDs in +// one database. +func TestABumpUnderOneStoreIDMovesTheEpochSeenThroughEveryStoreID(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + const one, two = "epoch-store-one", "epoch-store-two" + + for _, bump := range []struct { + name string + run func() (int, error) + }{ + {"BumpEpoch", func() (int, error) { return te.store.BumpEpoch(ctx, one, "restore") }}, + {"BumpEpochCarrying", func() (int, error) { return te.store.BumpEpochCarrying(ctx, one, "token-scheme-change") }}, + } { + before, err := te.store.CurrentEpoch(ctx, two) + if err != nil { + t.Fatalf("CurrentEpoch(%s): %v", two, err) + } + after, err := bump.run() + if err != nil { + t.Fatalf("%s(%s): %v", bump.name, one, err) + } + seen, err := te.store.CurrentEpoch(ctx, two) + if err != nil { + t.Fatalf("CurrentEpoch(%s): %v", two, err) + } + if after != before+1 || seen != after { + t.Errorf("%s under %s moved the epoch from %d to %d, and CurrentEpoch under %s reads %d: the counter is per database, so one bump advances the epoch every storeID sees", bump.name, one, before, after, two, seen) + } + } +} + +// epochMint mints id under store, failing the test if the mint fails. +func epochMint(t *testing.T, ctx context.Context, te *testEnv, store, id string) string { + t.Helper() + address, err := te.store.MintUnderEpoch(ctx, store, id) + if err != nil { + t.Fatalf("MintUnderEpoch(%s, %s): %v", store, id, err) + } + return address +} + +// epochBump moves the epoch with the counter-only bump (a restore, which +// writes no address rows) and returns the new epoch. +func epochBump(t *testing.T, ctx context.Context, te *testEnv, store string) int { + t.Helper() + epoch, err := te.store.BumpEpoch(ctx, store, "restore") + if err != nil { + t.Fatalf("BumpEpoch(%s): %v", store, err) + } + return epoch +} + +// epochCarry runs the token-scheme-change bump, which also carries every live +// address to the new epoch, and returns the new epoch. +func epochCarry(t *testing.T, ctx context.Context, te *testEnv, store string) int { + t.Helper() + epoch, err := te.store.BumpEpochCarrying(ctx, store, "token-scheme-change") + if err != nil { + t.Fatalf("BumpEpochCarrying(%s): %v", store, err) + } + return epoch +} + +// epochRowsAt counts the rows of store that were minted at epoch. After a +// carrying bump to epoch that is the number of carry rows it wrote, plus any +// row the test minted there itself. +func epochRowsAt(t *testing.T, ctx context.Context, te *testEnv, store string, epoch int) int { + t.Helper() + var count int + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM epoch_minted_addresses WHERE store_id = ? AND minted_epoch = ?", + []any{store, epoch}, &count) + return count +} + +// epochRowAddress returns the address of the one row of id in store minted at +// epoch, failing the test unless there is exactly one. +func epochRowAddress(t *testing.T, ctx context.Context, te *testEnv, store, id string, epoch int) string { + t.Helper() + var count int + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM epoch_minted_addresses WHERE store_id = ? AND minted_id = ? AND minted_epoch = ?", + []any{store, id, epoch}, &count) + if count != 1 { + t.Fatalf("store %s holds %d rows for %s minted at epoch %d, want exactly 1", store, count, id, epoch) + } + var address string + te.queryScalar(t, ctx, "SELECT address FROM epoch_minted_addresses WHERE store_id = ? AND minted_id = ? AND minted_epoch = ?", + []any{store, id, epoch}, &address) + return address +} + +// epochCarriedFrom reads address's carried_from: not Valid for a row that was +// minted fresh. +func epochCarriedFrom(t *testing.T, ctx context.Context, te *testEnv, address string) sql.NullString { + t.Helper() + var carriedFrom sql.NullString + te.queryScalar(t, ctx, "SELECT carried_from FROM epoch_minted_addresses WHERE address = ?", + []any{address}, &carriedFrom) + return carriedFrom +} + +// epochGoneAt reads address's gone_at_epoch: not Valid while the address is +// live. +func epochGoneAt(t *testing.T, ctx context.Context, te *testEnv, address string) sql.NullInt64 { + t.Helper() + var goneAt sql.NullInt64 + te.queryScalar(t, ctx, "SELECT gone_at_epoch FROM epoch_minted_addresses WHERE address = ?", + []any{address}, &goneAt) + return goneAt +} + +// epochCarryRowsOf counts the carry rows that name root as their lineage root. +func epochCarryRowsOf(t *testing.T, ctx context.Context, te *testEnv, root string) int { + t.Helper() + var count int + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM epoch_minted_addresses WHERE carried_from = ?", + []any{root}, &count) + return count +} + +// epochRequireServed fails unless the store serves address: StillServes is true +// and Resolve answers Live. why says what the assertion checks. +func epochRequireServed(t *testing.T, ctx context.Context, te *testEnv, store, address, why string) { + t.Helper() + serves, err := te.store.StillServes(ctx, store, address) + if err != nil { + t.Fatalf("StillServes(%s): %v", address, err) + } + if !serves { + t.Errorf("StillServes(%s) = false, want true: %s", address, why) + } + result, err := te.store.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s): %v", address, err) + } + if result.Restriction != storeops.EpochRestrictionLive { + t.Errorf("Resolve(%s).Restriction = %d, want Live (%d): %s", address, result.Restriction, storeops.EpochRestrictionLive, why) + } +} + +// epochRequireGone fails unless address is no longer served at epoch: +// StillServes is false and Resolve answers GoneReorganization naming epoch. +func epochRequireGone(t *testing.T, ctx context.Context, te *testEnv, store, address string, epoch int, why string) { + t.Helper() + serves, err := te.store.StillServes(ctx, store, address) + if err != nil { + t.Fatalf("StillServes(%s): %v", address, err) + } + if serves { + t.Errorf("StillServes(%s) = true, want false: %s", address, why) + } + result, err := te.store.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s): %v", address, err) + } + if result.Restriction != storeops.EpochRestrictionGoneReorganization { + t.Errorf("Resolve(%s).Restriction = %d, want GoneReorganization (%d): %s", address, result.Restriction, storeops.EpochRestrictionGoneReorganization, why) + } + if result.Epoch == nil { + t.Errorf("Resolve(%s).Epoch = nil, want the current epoch %d: %s", address, epoch, why) + } else if *result.Epoch != epoch { + t.Errorf("Resolve(%s).Epoch = %d, want the current epoch %d: %s", address, *result.Epoch, epoch, why) + } +} diff --git a/internal/storage/embeddeddolt/expected_revision_contract_test.go b/internal/storage/embeddeddolt/expected_revision_contract_test.go index a716384f6..6db4e2f0e 100644 --- a/internal/storage/embeddeddolt/expected_revision_contract_test.go +++ b/internal/storage/embeddeddolt/expected_revision_contract_test.go @@ -10,11 +10,21 @@ import ( ) // TestExpectedRevisionContract wires this leg into the R16/R17 -// expected-revision contract. Phase 0 leaves every hook nil in every -// backend's fixture kit (architecture §12), so each case below skips by -// name; this file exists so TestEveryLegWiresEveryRoleContract counts this -// leg, and so the cases start running for real the moment this leg's fixture -// kit grows a non-nil hook. +// expected-revision contract. Every hook is nil (architecture §12's Phase 0 +// default), so each case below skips by name; this file exists so +// TestEveryLegWiresEveryRoleContract counts this leg, and so the cases +// start running for real the moment this leg grows a real +// CompareAndSetVersion (gastownhall/beads#6358). +// +// This leg previously wired every hook to an honest "not implemented" stub +// instead of leaving them nil, so each case ran for real and +// failed for that documented reason rather than skipping. That premise — +// that a later slice would give this leg its CompareAndSetVersion — went +// stale once review moved the real implementation to +// gastownhall/beads#6358 instead; a stub whose failure message blames a +// bead that will never land it is worse than an honest skip, so this file +// reverts to nil hooks (gastownhall/beads#6664, bee-ghosttrack review +// 5268699223). func TestExpectedRevisionContract(t *testing.T) { ctx := context.Background() fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "erev"} diff --git a/internal/storage/embeddeddolt/retention_epoch_contract_test.go b/internal/storage/embeddeddolt/retention_epoch_contract_test.go index 648341895..6c9a7b52e 100644 --- a/internal/storage/embeddeddolt/retention_epoch_contract_test.go +++ b/internal/storage/embeddeddolt/retention_epoch_contract_test.go @@ -4,9 +4,12 @@ package embeddeddolt_test import ( "context" + "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + storeops "github.com/steveyegge/beads/internal/storage/issueops" ) // TestRetentionContract wires this leg into the R20 retention contract. @@ -59,11 +62,31 @@ func TestRetentionContract(t *testing.T) { }) } -// TestEpochContract wires this leg into the R20 epoch contract. Same Phase 0 -// all-nil state as TestRetentionContract above. +// TestEpochContract runs the R20 epoch contract (gastownhall/beads#5898 +// revision 9, #6136) against the embedded-Dolt- +// backed store, which reaches internal/storage/issueops's epoch Tx +// functions through this leg's own issue-operation transaction +// (EmbeddedDoltStore.BumpEpoch/BumpEpochCarrying/MintUnderEpoch/LoseVersion) +// or a read-only connection (EmbeddedDoltStore.CurrentEpoch/StillServes/ +// Resolve/CurrentAddressFor) — see internal/storage/embeddeddolt/epoch_cas.go. +// +// All three legs run that one shared body, so this is not an independent +// vote on the design — it is the check on THIS leg's wrapper and this file's +// own conformance-vocabulary translation. func TestEpochContract(t *testing.T) { - ctx := context.Background() - fixture := conformance.EpochFixture{IssuePrefix: "epch"} + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "epch") + ctx := t.Context() + fixture := conformance.EpochFixture{ + IssuePrefix: "epch", + CurrentEpoch: epochEmbeddedCurrentEpoch(te.store), + BumpEpoch: epochEmbeddedBumpEpoch(te.store), + MintUnderEpoch: epochEmbeddedMintUnderEpoch(te.store), + StillServes: epochEmbeddedStillServes(te.store), + Resolve: epochEmbeddedResolve(te.store), + CurrentAddressFor: epochEmbeddedCurrentAddressFor(te.store), + LoseVersion: epochEmbeddedLoseVersion(te.store), + } t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { conformance.RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) @@ -71,4 +94,98 @@ func TestEpochContract(t *testing.T) { t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { conformance.RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) }) + t.Run("TokenSchemeChangeCarriesLiveAddressesAndOnlyThose", func(t *testing.T) { + conformance.RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose(t, ctx, fixture) + }) + t.Run("ACarriedAddressKeepsResolvingAcrossLaterBumps", func(t *testing.T) { + conformance.RunACarriedAddressKeepsResolvingAcrossLaterBumps(t, ctx, fixture) + }) + t.Run("ALostAddressStaysGoneAcrossASchemeChange", func(t *testing.T) { + conformance.RunALostAddressStaysGoneAcrossASchemeChange(t, ctx, fixture) + }) +} + +func epochEmbeddedCurrentEpoch(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID string) (int, error) { + return func(ctx context.Context, storeID string) (int, error) { + return store.CurrentEpoch(ctx, storeID) + } +} + +// epochEmbeddedBumpEpoch chooses the bump by the typed trigger: only a +// token-scheme-change bump carries live addresses to the new epoch, and the +// other two triggers move the counter alone. The choice lives here, in the +// contract-test translation layer, because the shared body has no switch on +// the reason string. +func epochEmbeddedBumpEpoch(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID string, trigger conformance.EpochBumpTrigger) (int, error) { + return func(ctx context.Context, storeID string, trigger conformance.EpochBumpTrigger) (int, error) { + if trigger == conformance.EpochBumpTriggerTokenSchemeChange { + return store.BumpEpochCarrying(ctx, storeID, trigger.String()) + } + return store.BumpEpoch(ctx, storeID, trigger.String()) + } +} + +func epochEmbeddedMintUnderEpoch(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID, id string) (conformance.Address, error) { + return func(ctx context.Context, storeID, id string) (conformance.Address, error) { + address, err := store.MintUnderEpoch(ctx, storeID, id) + if err != nil { + return "", err + } + return conformance.Address(address), nil + } +} + +func epochEmbeddedStillServes(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID string, address conformance.Address) (bool, error) { + return func(ctx context.Context, storeID string, address conformance.Address) (bool, error) { + return store.StillServes(ctx, storeID, string(address)) + } +} + +func epochEmbeddedResolve(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID string, address conformance.Address) (conformance.RetentionAnswer, error) { + return func(ctx context.Context, storeID string, address conformance.Address) (conformance.RetentionAnswer, error) { + result, err := store.Resolve(ctx, storeID, string(address)) + if err != nil { + return conformance.RetentionAnswer{}, err + } + return epochResolveResultToConformance(result), nil + } +} + +func epochEmbeddedCurrentAddressFor(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID string, oldAddress conformance.Address) (conformance.Address, error) { + return func(ctx context.Context, storeID string, oldAddress conformance.Address) (conformance.Address, error) { + address, err := store.CurrentAddressFor(ctx, storeID, string(oldAddress)) + if err != nil { + return "", err + } + return conformance.Address(address), nil + } +} + +func epochEmbeddedLoseVersion(store *embeddeddolt.EmbeddedDoltStore) func(ctx context.Context, storeID string, address conformance.Address) error { + return func(ctx context.Context, storeID string, address conformance.Address) error { + return store.LoseVersion(ctx, storeID, string(address)) + } +} + +// epochRestrictionToConformance translates this leg's local +// storeops.EpochRestriction into backend/conformance's own Restriction +// vocabulary — the contract-test file is the translation boundary, per +// epoch_cas.go's package doc. +func epochRestrictionToConformance(r storeops.EpochRestriction) conformance.Restriction { + switch r { + case storeops.EpochRestrictionLive: + return conformance.RestrictionLive + case storeops.EpochRestrictionGoneReorganization: + return conformance.RestrictionGoneReorganization + default: + return conformance.RestrictionUnknown + } +} + +func epochResolveResultToConformance(r storeops.EpochResolveResult) conformance.RetentionAnswer { + return conformance.RetentionAnswer{ + Restriction: epochRestrictionToConformance(r.Restriction), + ProducingStore: r.ProducingStore, + Epoch: r.Epoch, + } } diff --git a/internal/storage/embeddeddolt/transaction.go b/internal/storage/embeddeddolt/transaction.go index ed1d25682..a9eb77f3a 100644 --- a/internal/storage/embeddeddolt/transaction.go +++ b/internal/storage/embeddeddolt/transaction.go @@ -50,6 +50,7 @@ func (s *EmbeddedDoltStore) runTransactionWithMessage(ctx context.Context, fn fu }); err != nil { return err } + afterSQLCommit() // Anything the body minted ran with versioned history scoped on this // store's transaction (withConn), so its version rows must be staged with @@ -67,6 +68,11 @@ func (s *EmbeddedDoltStore) runTransactionWithMessage(ctx context.Context, fn fu return nil } +// afterSQLCommit runs between runTransactionWithMessage's SQL commit and the +// Dolt commit that publishes it. It is a no-op outside tests, which use it to +// fail the publication of a write whose rows have already landed. +var afterSQLCommit = func() {} + func wrapCommitIndeterminate(op string, err error) error { return fmt.Errorf("%s: %w: %w", op, err, storage.ErrCommitIndeterminate) } diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 138b27d5b..3ade280ae 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -53,6 +53,7 @@ go_library( "edge_counts.go", "edges.go", "epic_closure.go", + "epoch_cas.go", "errors.go", "events.go", "execution.go", @@ -164,6 +165,7 @@ go_test( "dependency_tree_test.go", "edge_counts_test.go", "edge_role_requests_test.go", + "epoch_cas_test.go", "execution_test.go", "filters_test.go", "gate_fields_test.go", @@ -212,6 +214,7 @@ go_test( ], data = glob(["testdata/**"]) + [ ":go_srcs", + "//internal/storage/dolt:go_srcs", "//internal/storage/domain/db:go_srcs", "//internal/storage/schema:migrations", "//internal/storage/versioncontrolops:go_srcs", diff --git a/internal/storage/issueops/epoch_cas.go b/internal/storage/issueops/epoch_cas.go new file mode 100644 index 000000000..994334ac3 --- /dev/null +++ b/internal/storage/issueops/epoch_cas.go @@ -0,0 +1,680 @@ +package issueops + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "strings" + "time" + + "github.com/steveyegge/beads/internal/storage/dberrors" +) + +// This file implements R20 epoch-transition enforcement (gastownhall/beads#5898 +// revision 9, #6136): a store-wide epoch generation +// counter (store_epoch, migration 0067) plus a durable record of the +// addresses minted under each generation (epoch_minted_addresses, migration +// 0072), used to answer whether a previously-minted address is still served +// by the store's current epoch. It adds no RetentionFixture/R17 resolve, +// remove, hold, force-remove, erase, or mint logic — R20 epoch reasoning is +// evaluated entirely on its own (out of scope for this slice). +// +// ALL THREE LEGS SHARE THIS BODY, the same way they share +// CompareAndSetMetadataKeyInTx: the two Dolt-backed stores wrap it in their +// own transaction, and the unit-of-work provider reaches it through its own +// leg-specific adapter. Each leg's own contract-test file is the +// translation boundary to and from backend/conformance's +// Address/EpochBumpTrigger/RetentionAnswer vocabulary — this file knows +// nothing about the conformance package. +// +// ADDRESSES ARE NEVER RECOMPUTED FROM store_epoch. Each is a deterministic +// token over (storeID, id, the epoch current at mint time) — see +// epochAddress — persisted once at mint: a later mint of the same id under a +// bumped epoch produces a DIFFERENT address and a new row, and every +// address's row survives to answer StillServes/Resolve after the epoch moves +// on. +// +// SURVIVAL IS RECORDED ON THE ROW, NEVER INFERRED (gastownhall/beads#6664, +// bee-ghosttrack review 5360880888, Major 1). An address is served unless its +// own row says gone_at_epoch, and a row minted in an epoch that has not +// happened yet is not served: the answer is a function of the address's row +// and the current epoch, and nothing reads another row to reach it. Gone is +// terminal. Two writers set gone_at_epoch, both guarded by IS NULL so that +// nothing already lost is touched and nothing is ever cleared: +// LoseVersionInTx, for a Version the store lost, and BumpEpochCarryingInTx, +// for the losing lineage when two live lineages of one id meet at a +// token-scheme change. +// +// A restore or a destructive reinit moves the counter and writes no address +// row (BumpEpochInTx), so every address that was served stays Live at its own +// address with no re-mint. A token-scheme change is the one bump that writes +// address rows (BumpEpochCarryingInTx): it carries every live lineage to the +// new epoch with one carry row whose carried_from names the lineage's ROOT +// address, never the previous carry, so lineages stay flat and any member of +// one reaches its newest address in a single indexed lookup. +// +// THE EPOCH COUNTER IS PER DATABASE, NOT PER STORE (gastownhall/beads#6664, +// bee-ghosttrack review 5360880888, Minor 3). store_epoch has no store_id +// column, and storeID appears only in error text: one bump advances the epoch +// seen through every storeID that shares the database. There is deliberately +// no storeID guard in this body, because the conformance suite uses several +// synthetic storeIDs in one database. +// +// epochAddress'S ENCODING IS LENGTH-PREFIXED, NOT BARE-COLON-JOINED +// (gastownhall/beads#6664, bee-ghosttrack review 5268699223, item B2): a +// colon inside storeID or id can no longer shift the (storeID, id, epoch) +// field boundary onto a different triple the way the old "epch:%s:%s:%d" +// form allowed. MintUnderEpochInTx additionally refuses a storeID or id +// containing ":" outright (validateEpochAddressInputs) so an address stays +// readable by eye without needing to count length prefixes — belt and +// braces, not a correctness dependency of epochAddress itself. Given that, +// upsertEpochMintedAddressInTx's exists branch is a pure no-op: an +// address's row existing now provably means its stored triple already +// matches the incoming one, so there is nothing left to write. + +// EpochRestriction is this file's own local answer vocabulary for R20 — +// deliberately not backend/conformance.Restriction (see package doc above). +// It omits GoneRetention/GoneErasure: those are RetentionFixture/R17 +// outcomes this slice does not produce. +type EpochRestriction int + +const ( + EpochRestrictionLive EpochRestriction = iota + EpochRestrictionGoneReorganization + EpochRestrictionUnknown +) + +// EpochResolveResult is ResolveEpochInTx's answer. Epoch is non-nil exactly +// when the answer required comparing an epoch (Live and GoneReorganization); +// it is nil for Unknown, which never looks store_epoch up at all. +type EpochResolveResult struct { + Restriction EpochRestriction + ProducingStore string + Epoch *int +} + +// ErrEpochAddressNotFound means the address named was never minted by +// storeID — distinct from an ordinary "gone" answer, which requires having +// minted the address in the first place. +var ErrEpochAddressNotFound = errors.New("epoch CAS: address not minted by this store") + +// ErrEpochAddressNotServed means the address exists in this store but the +// store does not serve it: its lineage was lost, or it was minted in an epoch +// that has not happened yet. It is distinct from ErrEpochAddressNotFound so a +// caller can tell an address the store never minted from one it minted and no +// longer serves. +var ErrEpochAddressNotServed = errors.New("epoch CAS: address exists in this store but is not served") + +// ErrEpochAddressSeparator means a storeID or id passed to +// MintUnderEpochInTx contains ":", the character epochAddress's encoding +// uses as a field delimiter (gastownhall/beads#6664, bee-ghosttrack review +// 5268699223, item B2b). epochAddress's length-prefixed encoding does not +// actually depend on this for correctness (see epochAddress's own doc +// comment), but refusing it outright keeps a minted address readable by eye. +var ErrEpochAddressSeparator = errors.New("epoch CAS: storeID or id contains \":\", epochAddress's field separator") + +// ErrEpochAddressTooLong means a storeID and id compose an address that +// epoch_minted_addresses.address, a VARCHAR(255) PRIMARY KEY (migration 0072), +// may not hold. MintUnderEpochInTx refuses such a pair before it reads or +// writes anything, so the caller gets this error instead of a raw driver error +// from the INSERT. The bound is charged at the widest epoch the INT counter can +// reach, not the current one: a token-scheme change re-encodes every live pair +// at the new epoch (BumpEpochCarryingInTx), so a pair that fit at epoch 9 must +// still fit at epoch 10, or the carry, and with it every later scheme change, +// would fail database-wide. +var ErrEpochAddressTooLong = errors.New("epoch CAS: storeID and id compose an address longer than epoch_minted_addresses.address holds") + +// maxEpochAddressLen is the width of epoch_minted_addresses.address, +// VARCHAR(255). The check against it counts bytes, which are never fewer than +// the characters the column counts, so it refuses no later than the column +// would. +const maxEpochAddressLen = 255 + +// epochAddress is a minted address: a deterministic token over storeID, id, +// and the epoch current at mint time, so re-minting the same id under the +// same epoch always reproduces the same address. storeID and id are +// length-prefixed, not just colon-joined (gastownhall/beads#6664, +// bee-ghosttrack review 5268699223, item B2a): reading exactly len(storeID) +// bytes for the first field and len(id) bytes for the second makes the +// (storeID, id) boundary unambiguous no matter what characters either one +// contains, so two different triples can never collide on the same address. +// MintUnderEpochInTx also refuses a storeID or id containing ":" outright +// (validateEpochAddressInputs, item B2b) so an address stays readable by +// eye without relying on that. +func epochAddress(storeID, id string, epoch int) string { + return fmt.Sprintf("epch:%d:%s:%d:%s:%d", len(storeID), storeID, len(id), id, epoch) +} + +// validateEpochAddressInputs rejects a storeID or id containing ":" before +// MintUnderEpochInTx mints an address from them (item B2b above), and a pair +// whose address would not fit the address column at the widest epoch +// (ErrEpochAddressTooLong). +func validateEpochAddressInputs(storeID, id string) error { + if strings.Contains(storeID, ":") { + return fmt.Errorf("%w: storeID %q", ErrEpochAddressSeparator, storeID) + } + if strings.Contains(id, ":") { + return fmt.Errorf("%w: id %q", ErrEpochAddressSeparator, id) + } + if n := len(epochAddress(storeID, id, math.MaxInt32)); n > maxEpochAddressLen { + return fmt.Errorf("%w: %d bytes at the widest epoch, limit %d", ErrEpochAddressTooLong, n, maxEpochAddressLen) + } + return nil +} + +// ensureStoreEpochRow lazily initializes store_epoch's singleton row +// (migration 0067; the table starts empty) to epoch 1 on first use, and +// reports the current epoch either way. Only BumpEpochInTx and +// MintUnderEpochInTx call this: both are about to write regardless, so +// creating the row here is not a surprising extra side effect. Every +// read-only path calls readStoreEpochWithoutSeedingInTx instead +// (gastownhall/beads#6664, bee-ghosttrack review 5268699223, item B4): a write +// from what callers reasonably expect to be a read is surprising at best, and +// a hard failure against a genuinely read-only connection at worst. +// +// The seed follows version_history.go's seed of the same row (gastownhall/ +// beads#6664, bee-ghosttrack review 5360880888): read first, so the INSERT +// happens once per store rather than once per mint; INSERT IGNORE, so a row +// that turns up between the read and the write is not an error; and read the +// row back, so the epoch reported is what the row holds even when the seed was +// ignored, not the 1 it would have written. +func ensureStoreEpochRow(ctx context.Context, tx DBTX) (int, error) { + var epoch int + err := tx.QueryRowContext(ctx, `SELECT epoch FROM store_epoch WHERE id = 1`).Scan(&epoch) + if errors.Is(err, sql.ErrNoRows) { + if _, err := tx.ExecContext(ctx, `INSERT IGNORE INTO store_epoch (id, epoch) VALUES (1, 1)`); err != nil { + return 0, fmt.Errorf("initialize store_epoch: %w", err) + } + err = tx.QueryRowContext(ctx, `SELECT epoch FROM store_epoch WHERE id = 1`).Scan(&epoch) + } + if err != nil { + return 0, fmt.Errorf("read store_epoch: %w", err) + } + return epoch, nil +} + +// EpochMintDirtyTables names the tables a mint (MintUnderEpochInTx) can leave +// modified, for the leg that publishes them to Dolt history. A token-scheme +// bump (BumpEpochCarryingInTx) leaves the same two tables modified. store_epoch is in the set although a mint +// never updates it: ensureStoreEpochRow INSERTs the singleton row on first use, +// so a mint that published only epoch_minted_addresses would leave that row +// modified and uncommitted, and the Dolt commit would hold minted rows without +// the epoch row they depend on (gastownhall/beads#6664 Major 2, bee-ghosttrack +// review 5360880888). Staging a table that turns out to be clean is free. +func EpochMintDirtyTables() ChangedTables { + return ChangedTables{"epoch_minted_addresses": true, "store_epoch": true} +} + +// readStoreEpochWithoutSeedingInTx reports the current epoch without writing: +// store_epoch's singleton row starting absent (migration 0067) and its +// epoch being 1 are the same state, so a read-only caller can answer from +// that default instead of initializing the row the way ensureStoreEpochRow +// does (item B4 above). It is deliberately not version_history.go's +// readStoreEpochInTx, which seeds the row when it is absent. CurrentEpochInTx, +// StillServesInTx, ResolveEpochInTx, CurrentAddressForInTx and +// LoseVersionInTx all read this way; only BumpEpochInTx and +// MintUnderEpochInTx, which are about to write regardless, use +// ensureStoreEpochRow. +func readStoreEpochWithoutSeedingInTx(ctx context.Context, tx DBTX) (int, error) { + var epoch int + err := tx.QueryRowContext(ctx, `SELECT epoch FROM store_epoch WHERE id = 1`).Scan(&epoch) + if errors.Is(err, sql.ErrNoRows) { + return 1, nil + } + if err != nil { + return 0, fmt.Errorf("read store_epoch: %w", err) + } + return epoch, nil +} + +// epochMintedAddress is one row as read from epoch_minted_addresses. +// carriedFrom is NULL for a root row (a fresh mint) and otherwise the address of +// the lineage's root; goneAtEpoch is NULL while the store serves the row and +// otherwise the epoch it was lost in. +type epochMintedAddress struct { + storeID string + mintedID string + mintedEpoch int + carriedFrom sql.NullString + goneAtEpoch sql.NullInt64 +} + +// served reports whether the store serves the row at epoch. A row is served +// unless it was lost, and a row minted in an epoch that has not happened yet +// fails closed rather than being trusted. +func (r epochMintedAddress) served(epoch int) bool { + return !r.goneAtEpoch.Valid && r.mintedEpoch <= epoch +} + +// lineageRoot is the address every member of the row's lineage names: the +// row's carried_from when it is a carry row, and its own address when it is a +// root. +func (r epochMintedAddress) lineageRoot(address string) string { + if r.carriedFrom.Valid { + return r.carriedFrom.String + } + return address +} + +// readEpochMintedAddressInTx reads address's row, if any. found is false +// (with a zero-value row and nil error) when address was never minted — +// distinguishing "not found" from an actual query error is the caller's +// job, the same way readExpectedRevisionRowInTx's callers do it. +func readEpochMintedAddressInTx(ctx context.Context, tx DBTX, address string) (epochMintedAddress, bool, error) { + var row epochMintedAddress + err := tx.QueryRowContext(ctx, + `SELECT store_id, minted_id, minted_epoch, carried_from, gone_at_epoch FROM epoch_minted_addresses WHERE address = ?`, address, + ).Scan(&row.storeID, &row.mintedID, &row.mintedEpoch, &row.carriedFrom, &row.goneAtEpoch) + if errors.Is(err, sql.ErrNoRows) { + return epochMintedAddress{}, false, nil + } + if err != nil { + return epochMintedAddress{}, false, fmt.Errorf("epoch CAS: read minted address %s: %w", address, err) + } + return row, true, nil +} + +// upsertEpochMintedAddressInTx writes address's row. exists (from the +// caller's own prior read) says whether one already does: if so, this is a +// pure no-op (gastownhall/beads#6664, bee-ghosttrack review 5268699223, +// item B2c) — epochAddress is now injective (item B2a), so an existing row +// at this exact address provably already carries this exact (storeID, +// mintedID, mintedEpoch) triple, and the old exists-branch UPDATE (which +// had no store_id/minted_id guard) could only ever have mattered by +// rewriting a DIFFERENT triple's row out from under it on an address +// collision the new encoding no longer allows. +// +// A duplicate-key error from the INSERT is NOT success. It means a row the +// caller's read did not find: another transaction inserted it, and that +// transaction or a later one may already have lost it, so taking the error as +// the benign race of two minters inserting the same row (item B3) could hand +// back an address that reads Gone. The row is read again and trusted only as +// far as this transaction can see it: served at mintedEpoch, the mint stands, +// because the row is the one it would have inserted; present but not served, +// the mint fails with ErrEpochAddressNotServed, as it does when the caller's +// own read finds such a row; not visible at all, the mint fails with the +// duplicate-key error, and a new transaction, which reads the committed row, +// answers it. +// +// On Dolt, the only supported backend, the INSERT does not hit that error for +// a row committed after the transaction's snapshot: it checks the snapshot, +// and the two inserts of one address conflict at COMMIT instead (Error 1213), +// which withRetryTx answers by running the mint again against the committed +// row (TestAMintRacingTheMintAndLossOfItsAddressConflictsAtCommit in the dolt +// package). FOR UPDATE is a parse-only no-op there, so no locking read could +// see further than the re-read does. +func upsertEpochMintedAddressInTx(ctx context.Context, tx DBTX, address, storeID, mintedID string, mintedEpoch int, exists bool) error { + if exists { + return nil + } + _, err := tx.ExecContext(ctx, + `INSERT INTO epoch_minted_addresses (address, store_id, minted_id, minted_epoch, minted_at) VALUES (?, ?, ?, ?, ?)`, + address, storeID, mintedID, mintedEpoch, time.Now().UTC(), + ) + if err == nil { + return nil + } + if !dberrors.IsDuplicateKey(err) { + return fmt.Errorf("epoch CAS: insert minted address %s: %w", address, err) + } + row, found, readErr := readEpochMintedAddressInTx(ctx, tx, address) + switch { + case readErr != nil: + return errors.Join(fmt.Errorf("epoch CAS: insert minted address %s: %w", address, err), readErr) + case !found: + return fmt.Errorf("epoch CAS: insert minted address %s: another transaction holds a row at it that this one cannot read, so whether the store serves it is unknown: %w", address, err) + case !row.served(mintedEpoch): + return fmt.Errorf("epoch CAS: mint %s under epoch for %s: %w: %s", mintedID, storeID, ErrEpochAddressNotServed, address) + } + return nil +} + +// CurrentEpochInTx reports storeID's current epoch generation. store_epoch +// has no store_id column: migration 0067 established it as one physical +// counter per database, matching production reality (one logical store per +// database already) — storeID is accepted here only to satisfy +// EpochFixture's hook signature. +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +func CurrentEpochInTx(ctx context.Context, tx DBTX, storeID string) (int, error) { + epoch, err := readStoreEpochWithoutSeedingInTx(ctx, tx) + if err != nil { + return 0, fmt.Errorf("epoch CAS: current epoch for %s: %w", storeID, err) + } + return epoch, nil +} + +// BumpEpochInTx advances storeID's epoch generation by one and records why +// (R20-a: a bump is triggered only by restore, destructive-reinit, or +// token-scheme-change — the leg adapter converts the fixture's +// conformance.EpochBumpTrigger to this plain string via trigger.String(), +// keeping that vocabulary out of this file per the package doc above). +// +// It writes store_epoch and nothing else: a restore or a destructive reinit +// leaves every address row alone, so every address that was served stays Live +// at its own address. A token-scheme change, which does write address rows, +// calls BumpEpochCarryingInTx instead. +// +// The epoch counter is per database, not per store. storeID appears only in +// error text; one bump advances the epoch seen through every storeID that +// shares this database. +func BumpEpochInTx(ctx context.Context, tx DBTX, storeID, reason string) (int, error) { + if _, err := ensureStoreEpochRow(ctx, tx); err != nil { + return 0, fmt.Errorf("epoch CAS: bump epoch for %s: %w", storeID, err) + } + if _, err := tx.ExecContext(ctx, + `UPDATE store_epoch SET epoch = epoch + 1, bumped_at = ?, bumped_reason = ? WHERE id = 1`, + time.Now().UTC(), reason, + ); err != nil { + return 0, fmt.Errorf("epoch CAS: bump epoch for %s: %w", storeID, err) + } + var newEpoch int + if err := tx.QueryRowContext(ctx, `SELECT epoch FROM store_epoch WHERE id = 1`).Scan(&newEpoch); err != nil { + return 0, fmt.Errorf("epoch CAS: read epoch after bump for %s: %w", storeID, err) + } + return newEpoch, nil +} + +// BumpEpochCarryingInTx is BumpEpochInTx for a token-scheme change, the one +// trigger that changes how addresses are encoded. It advances the epoch exactly +// as BumpEpochInTx does and also records the carry: every lineage the store +// still serves gets one new row at the new epoch, whose address is the +// deterministic token for (store, id, new epoch) and whose carried_from names +// the lineage's ROOT address. The old addresses keep resolving, and +// CurrentAddressForInTx reports the new one from any member of the lineage. +// +// The carry is DATABASE-WIDE on purpose. The counter is per database, so this +// bump moves the epoch every storeID sees, and carrying only storeID's rows +// would strand every other store's addresses at the scheme boundary. storeID is +// used for error text only, exactly as in BumpEpochInTx. It costs one read of +// every live row and one insert per live id, in one transaction: acceptable +// because a scheme change is a once-per-change migration, not a runtime path. +// +// Two live lineages of one id cannot both receive an address, because an +// address is a function of (store, id, epoch) and so is one per id per epoch. +// The lineage holding the newest row is carried, and every other lineage of +// that id is marked gone at the new epoch (the safe direction: over-void, never +// revive). Lineages already lost are not read, so they are never carried, and a +// row minted in an epoch that has not happened yet is not read either: it stays +// not served. A row already sitting at a carry's address means the counter +// rewound, so that is a failure of the bump: no re-read can settle it the way +// a mint settles a duplicate key (upsertEpochMintedAddressInTx). +// +// CALLERS MUST HOLD THE WORKSPACE GATE (internal/workspacegate) EXCLUSIVELY FOR +// THE BUMP, as a restore does. This is the one epoch operation that needs +// mints excluded by contract: the store cannot make a mint conflict with a bump +// (a mint has no cell of its own to change), so a mint that commits after this +// bump read the live rows would leave a live row that was never carried. +func BumpEpochCarryingInTx(ctx context.Context, tx DBTX, storeID, reason string) (int, error) { + newEpoch, err := BumpEpochInTx(ctx, tx, storeID, reason) + if err != nil { + return 0, err + } + candidates, err := readEpochCarryCandidatesInTx(ctx, tx, newEpoch) + if err != nil { + return 0, fmt.Errorf("epoch CAS: carry addresses to epoch %d for %s: %w", newEpoch, storeID, err) + } + + // The candidates are ordered newest first within each (store, id), so the + // first row of a group holds the lineage that wins it. + type lineageKey struct{ storeID, mintedID string } + var groups []lineageKey + winningRoot := map[lineageKey]string{} + type losingLineage struct{ storeID, root string } + var losers []losingLineage + seenLoser := map[losingLineage]bool{} + for _, c := range candidates { + key := lineageKey{c.storeID, c.mintedID} + winner, seen := winningRoot[key] + if !seen { + winningRoot[key] = c.root() + groups = append(groups, key) + continue + } + if loser := (losingLineage{c.storeID, c.root()}); c.root() != winner && !seenLoser[loser] { + seenLoser[loser] = true + losers = append(losers, loser) + } + } + + for _, loser := range losers { + if err := loseLineageInTx(ctx, tx, loser.storeID, loser.root, newEpoch); err != nil { + return 0, fmt.Errorf("epoch CAS: carry addresses to epoch %d for %s: %w", newEpoch, storeID, err) + } + } + for _, key := range groups { + address := epochAddress(key.storeID, key.mintedID, newEpoch) + if _, err := tx.ExecContext(ctx, + `INSERT INTO epoch_minted_addresses (address, store_id, minted_id, minted_epoch, minted_at, carried_from) VALUES (?, ?, ?, ?, ?, ?)`, + address, key.storeID, key.mintedID, newEpoch, time.Now().UTC(), winningRoot[key], + ); err != nil { + return 0, fmt.Errorf("epoch CAS: carry %s to epoch %d for %s: %w", key.mintedID, newEpoch, storeID, err) + } + } + return newEpoch, nil +} + +// epochCarryCandidate is one live row below the new epoch, as +// BumpEpochCarryingInTx reads it. +type epochCarryCandidate struct { + address string + storeID string + mintedID string + carriedFrom sql.NullString +} + +// root is the address of the candidate's lineage root. +func (c epochCarryCandidate) root() string { + if c.carriedFrom.Valid { + return c.carriedFrom.String + } + return c.address +} + +// readEpochCarryCandidatesInTx reads every row the store still serves below +// newEpoch, newest first within each (store, id). Rows already lost are not +// read, and neither are rows minted at or above newEpoch: those are anomalies +// and stay not served. The result is read to the end before any other statement +// runs on the transaction. +func readEpochCarryCandidatesInTx(ctx context.Context, tx DBTX, newEpoch int) ([]epochCarryCandidate, error) { + rows, err := tx.QueryContext(ctx, + `SELECT address, store_id, minted_id, carried_from FROM epoch_minted_addresses WHERE gone_at_epoch IS NULL AND minted_epoch < ? ORDER BY store_id, minted_id, minted_epoch DESC`, + newEpoch, + ) + if err != nil { + return nil, fmt.Errorf("read live addresses: %w", err) + } + defer func() { _ = rows.Close() }() + var candidates []epochCarryCandidate + for rows.Next() { + var c epochCarryCandidate + if err := rows.Scan(&c.address, &c.storeID, &c.mintedID, &c.carriedFrom); err != nil { + return nil, fmt.Errorf("scan live address: %w", err) + } + candidates = append(candidates, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read live addresses: %w", err) + } + return candidates, nil +} + +// loseLineageInTx marks the lineage rooted at root gone at epoch: the root's own +// row, then every carry row that names it. Each statement is guarded by +// gone_at_epoch IS NULL, so a row already lost is never touched and nothing is +// ever cleared. It is the only place gone_at_epoch is written, shared by +// LoseVersionInTx and BumpEpochCarryingInTx. +func loseLineageInTx(ctx context.Context, tx DBTX, storeID, root string, epoch int) error { + if _, err := tx.ExecContext(ctx, + `UPDATE epoch_minted_addresses SET gone_at_epoch = ? WHERE address = ? AND store_id = ? AND gone_at_epoch IS NULL`, + epoch, root, storeID, + ); err != nil { + return fmt.Errorf("mark %s lost at epoch %d: %w", root, epoch, err) + } + if _, err := tx.ExecContext(ctx, + `UPDATE epoch_minted_addresses SET gone_at_epoch = ? WHERE carried_from = ? AND store_id = ? AND gone_at_epoch IS NULL`, + epoch, root, storeID, + ); err != nil { + return fmt.Errorf("mark the carries of %s lost at epoch %d: %w", root, epoch, err) + } + return nil +} + +// MintUnderEpochInTx mints id's address under storeID's CURRENT epoch, +// deterministically (epochAddress): minting the same id again under the +// same epoch reproduces the same address and is an idempotent no-op upsert +// of the same row. +// +// A mint only ever INSERTS its own root row. It never updates another row, so +// no mint can retire or revive an address, and it never hands back an address +// the store does not serve: a row already sitting at the computed address is +// only possible when the lineage was minted in this epoch and lost in this same +// epoch, and the mint then returns ErrEpochAddressNotServed instead of an +// address that reads Gone. That includes a row another transaction inserts +// after this one read: on Dolt the two inserts conflict at COMMIT and the mint +// runs again against the committed row, and an INSERT that reports the row as +// a duplicate key reads it again first (upsertEpochMintedAddressInTx). +// +// A storeID or id containing ":" is refused (ErrEpochAddressSeparator), and so +// is a pair whose address could outgrow the address column at some epoch +// (ErrEpochAddressTooLong), before anything is read or written. +func MintUnderEpochInTx(ctx context.Context, tx DBTX, storeID, id string) (string, error) { + if err := validateEpochAddressInputs(storeID, id); err != nil { + return "", fmt.Errorf("epoch CAS: mint %s under epoch for %s: %w", id, storeID, err) + } + epoch, err := ensureStoreEpochRow(ctx, tx) + if err != nil { + return "", fmt.Errorf("epoch CAS: mint %s under epoch for %s: %w", id, storeID, err) + } + address := epochAddress(storeID, id, epoch) + row, found, err := readEpochMintedAddressInTx(ctx, tx, address) + if err != nil { + return "", err + } + if found && !row.served(epoch) { + return "", fmt.Errorf("epoch CAS: mint %s under epoch for %s: %w: %s", id, storeID, ErrEpochAddressNotServed, address) + } + if err := upsertEpochMintedAddressInTx(ctx, tx, address, storeID, id, epoch, found); err != nil { + return "", err + } + return address, nil +} + +// StillServesInTx reports whether address is still served under storeID's +// CURRENT epoch. The answer is read from the address's own row and the epoch and +// from nothing else: the row's gone_at_epoch says whether the store lost it, and +// a row minted in an epoch that has not happened yet is not served. An address +// that survived a restore or a destructive reinit therefore stays served with no +// upkeep, and an address from a different store, or one never minted, is not +// served either. +func StillServesInTx(ctx context.Context, tx DBTX, storeID, address string) (bool, error) { + row, found, err := readEpochMintedAddressInTx(ctx, tx, address) + if err != nil { + return false, err + } + if !found || row.storeID != storeID { + return false, nil + } + epoch, err := readStoreEpochWithoutSeedingInTx(ctx, tx) + if err != nil { + return false, fmt.Errorf("epoch CAS: still serves %s for %s: %w", address, storeID, err) + } + return row.served(epoch), nil +} + +// ResolveEpochInTx answers R20's epoch-only restriction for address: Live while +// the store serves it, GoneReorganization once the store lost it (a +// reorganization, not a retention or erasure outcome; RetentionFixture/R17 +// states are out of scope here), and Unknown for an address this store never +// minted. Like StillServesInTx it reads only the address's own row and the +// epoch. ProducingStore is always storeID: this file has no lineage/replica +// model to attribute a foreign store to (unlike RetentionFixture's cross-store +// answers). +func ResolveEpochInTx(ctx context.Context, tx DBTX, storeID, address string) (EpochResolveResult, error) { + row, found, err := readEpochMintedAddressInTx(ctx, tx, address) + if err != nil { + return EpochResolveResult{}, err + } + if !found || row.storeID != storeID { + return EpochResolveResult{Restriction: EpochRestrictionUnknown, ProducingStore: storeID}, nil + } + epoch, err := readStoreEpochWithoutSeedingInTx(ctx, tx) + if err != nil { + return EpochResolveResult{}, fmt.Errorf("epoch CAS: resolve %s for %s: %w", address, storeID, err) + } + if row.served(epoch) { + return EpochResolveResult{Restriction: EpochRestrictionLive, ProducingStore: storeID, Epoch: &epoch}, nil + } + return EpochResolveResult{Restriction: EpochRestrictionGoneReorganization, ProducingStore: storeID, Epoch: &epoch}, nil +} + +// CurrentAddressForInTx reports the address oldAddress now resolves to: the +// newest address of its lineage, or oldAddress itself when nothing carried the +// Version to a new address. It only reads. It executes no INSERT, UPDATE or +// DELETE, because a lookup that minted an address could make a lost address +// look served again (gastownhall/beads#6664, bee-ghosttrack review 5360880888, +// Major 1), and both Dolt legs run it on a read-write transaction that is +// always rolled back, so a stray write would vanish silently instead of +// failing. oldAddress must be one storeID minted (ErrEpochAddressNotFound +// otherwise) and still serves (ErrEpochAddressNotServed otherwise). +func CurrentAddressForInTx(ctx context.Context, tx DBTX, storeID, oldAddress string) (string, error) { + row, found, err := readEpochMintedAddressInTx(ctx, tx, oldAddress) + if err != nil { + return "", err + } + if !found || row.storeID != storeID { + return "", fmt.Errorf("epoch CAS: current address for %s: %w: %s", storeID, ErrEpochAddressNotFound, oldAddress) + } + epoch, err := readStoreEpochWithoutSeedingInTx(ctx, tx) + if err != nil { + return "", fmt.Errorf("epoch CAS: current address for %s: %w", storeID, err) + } + if !row.served(epoch) { + return "", fmt.Errorf("epoch CAS: current address for %s: %w: %s", storeID, ErrEpochAddressNotServed, oldAddress) + } + var newest string + err = tx.QueryRowContext(ctx, + `SELECT address FROM epoch_minted_addresses WHERE carried_from = ? AND store_id = ? AND gone_at_epoch IS NULL AND minted_epoch <= ? ORDER BY minted_epoch DESC LIMIT 1`, + row.lineageRoot(oldAddress), storeID, epoch, + ).Scan(&newest) + if errors.Is(err, sql.ErrNoRows) { + return oldAddress, nil + } + if err != nil { + return "", fmt.Errorf("epoch CAS: current address for %s: read the newest carry of %s: %w", storeID, oldAddress, err) + } + return newest, nil +} + +// LoseVersionInTx records that storeID no longer serves the Version named by +// address: it marks the lineage ROOT's own row and every carry row of that +// root gone at the current epoch, so every address of the Version resolves +// GoneReorganization from then on. Each statement is guarded by +// gone_at_epoch IS NULL, so nothing already lost is touched and nothing is ever +// cleared (Gone is terminal). Losing a lineage that is already lost matches no +// rows and returns nil, so a retried transaction may replay it. address must be +// one storeID minted, or ErrEpochAddressNotFound is returned and nothing is +// written. A lineage member minted in an epoch that has not happened yet +// violates the table's gone-not-before-mint check, and the resulting error is +// returned rather than papered over. +func LoseVersionInTx(ctx context.Context, tx DBTX, storeID, address string) error { + row, found, err := readEpochMintedAddressInTx(ctx, tx, address) + if err != nil { + return err + } + if !found || row.storeID != storeID { + return fmt.Errorf("epoch CAS: lose version for %s: %w: %s", storeID, ErrEpochAddressNotFound, address) + } + epoch, err := readStoreEpochWithoutSeedingInTx(ctx, tx) + if err != nil { + return fmt.Errorf("epoch CAS: lose version %s for %s: %w", address, storeID, err) + } + if err := loseLineageInTx(ctx, tx, storeID, row.lineageRoot(address), epoch); err != nil { + return fmt.Errorf("epoch CAS: lose version %s for %s: %w", address, storeID, err) + } + return nil +} diff --git a/internal/storage/issueops/epoch_cas_test.go b/internal/storage/issueops/epoch_cas_test.go new file mode 100644 index 000000000..223a40581 --- /dev/null +++ b/internal/storage/issueops/epoch_cas_test.go @@ -0,0 +1,1141 @@ +package issueops + +import ( + "context" + "database/sql/driver" + "errors" + "go/ast" + "go/parser" + "go/token" + "math" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +// These are sqlmock tests because each one pins a SHAPE: which statements the +// shared epoch body runs, in what order, and above all which statements it does +// not run. sqlmock rejects any statement without a registered expectation, so a +// test that registers no ExpectExec is a test that forbids every write. What a +// statement does to real rows (set semantics, carry, collision) is checked +// against a real store by the embedded leg's tests. +// +// The survival model these tests pin (gastownhall/beads#6664, bee-ghosttrack +// review 5360880888, Major 1): an address's status is a function of its OWN +// row and the current epoch. It is Live unless gone_at_epoch is set, and a row +// minted in an epoch that has not happened yet is not served. Nothing infers +// status from other rows. +const ( + epochSQLRow = `SELECT store_id, minted_id, minted_epoch, carried_from, gone_at_epoch FROM epoch_minted_addresses WHERE address = \?` + epochSQLReadEpoch = `SELECT epoch FROM store_epoch WHERE id = 1` + epochSQLNewest = `SELECT address FROM epoch_minted_addresses WHERE carried_from = \? AND store_id = \? AND gone_at_epoch IS NULL AND minted_epoch <= \? ORDER BY minted_epoch DESC LIMIT 1` + epochSQLLoseRoot = `UPDATE epoch_minted_addresses SET gone_at_epoch = \? WHERE address = \? AND store_id = \? AND gone_at_epoch IS NULL` + epochSQLLoseCarry = `UPDATE epoch_minted_addresses SET gone_at_epoch = \? WHERE carried_from = \? AND store_id = \? AND gone_at_epoch IS NULL` + epochSQLInsertRoot = `INSERT INTO epoch_minted_addresses \(address, store_id, minted_id, minted_epoch, minted_at\) VALUES \(\?, \?, \?, \?, \?\)` +) + +var epochSQLRowColumns = []string{"store_id", "minted_id", "minted_epoch", "carried_from", "gone_at_epoch"} + +// epochTestRow is one epoch_minted_addresses row as the shared body reads it. +type epochTestRow struct { + storeID string + mintedID string + mintedEpoch int + // carriedFrom is nil for a root row, else the lineage root's address. + carriedFrom any + // goneAtEpoch is nil while the lineage is served, else the epoch it was + // lost in. + goneAtEpoch any +} + +// expectEpochRow registers the read of address's row. A nil row is an address +// the store never minted. +func expectEpochRow(mock sqlmock.Sqlmock, address string, row *epochTestRow) { + rows := sqlmock.NewRows(epochSQLRowColumns) + if row != nil { + rows.AddRow(row.storeID, row.mintedID, row.mintedEpoch, row.carriedFrom, row.goneAtEpoch) + } + mock.ExpectQuery(epochSQLRow).WithArgs(address).WillReturnRows(rows) +} + +// expectEpochRead registers the read of store_epoch's singleton. It takes no +// arguments: the counter is per database, so no storeID ever reaches this SQL. +func expectEpochRead(mock sqlmock.Sqlmock, epoch int) { + mock.ExpectQuery(epochSQLReadEpoch). + WithArgs([]driver.Value{}...). + WillReturnRows(sqlmock.NewRows([]string{"epoch"}).AddRow(epoch)) +} + +// expectEpochAbsent registers the read of store_epoch's singleton finding no +// row, the state the table starts in (migration 0067). The other cases register +// a present row, where a reader that seeds the row and one that does not run the +// same SQL; only an absent row tells them apart. +func expectEpochAbsent(mock sqlmock.Sqlmock) { + mock.ExpectQuery(epochSQLReadEpoch). + WithArgs([]driver.Value{}...). + WillReturnRows(sqlmock.NewRows([]string{"epoch"})) +} + +func intPtr(v int) *int { return &v } + +// TestEpochReadPathsNeverSeedAnAbsentStoreEpochRow pins item B4 +// (gastownhall/beads#6664, bee-ghosttrack review 5268699223): every path that +// only reads answers an absent store_epoch row from the epoch-1 default and +// never inserts it. LoseVersionInTx is here too, because the epoch it reads is +// not one it writes. Each case registers no INSERT, so a read path switched to +// the seeding reader (ensureStoreEpochRow) runs an unexpected statement and +// fails; the Dolt legs cannot catch that switch, because their read +// transactions roll back. +func TestEpochReadPathsNeverSeedAnAbsentStoreEpochRow(t *testing.T) { + t.Parallel() + + const storeID = "absent-epoch-store" + root := epochAddress(storeID, "record-a", 1) + row := &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1} + ctx := context.Background() + + t.Run("CurrentEpochInTx", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochAbsent(mock) + got, err := CurrentEpochInTx(ctx, tx, storeID) + if err != nil || got != 1 { + t.Fatalf("CurrentEpochInTx with no store_epoch row = (%d, %v), want (1, nil)", got, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("CurrentEpochInTx must only read: %v", err) + } + }) + t.Run("StillServesInTx", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, root, row) + expectEpochAbsent(mock) + got, err := StillServesInTx(ctx, tx, storeID, root) + if err != nil || !got { + t.Fatalf("StillServesInTx with no store_epoch row = (%v, %v), want (true, nil): an address minted at epoch 1 is served at the default epoch", got, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("StillServesInTx must only read: %v", err) + } + }) + t.Run("ResolveEpochInTx", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, root, row) + expectEpochAbsent(mock) + got, err := ResolveEpochInTx(ctx, tx, storeID, root) + if err != nil || got.Restriction != EpochRestrictionLive || got.Epoch == nil || *got.Epoch != 1 { + t.Fatalf("ResolveEpochInTx with no store_epoch row = (%+v, %v), want Live at epoch 1", got, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("ResolveEpochInTx must only read: %v", err) + } + }) + t.Run("CurrentAddressForInTx", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, root, row) + expectEpochAbsent(mock) + mock.ExpectQuery(epochSQLNewest).WithArgs(root, storeID, int64(1)). + WillReturnRows(sqlmock.NewRows([]string{"address"})) + got, err := CurrentAddressForInTx(ctx, tx, storeID, root) + if err != nil || got != root { + t.Fatalf("CurrentAddressForInTx with no store_epoch row = (%q, %v), want (%q, nil)", got, err, root) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("CurrentAddressForInTx must only read: %v", err) + } + }) + t.Run("LoseVersionInTx", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, root, row) + expectEpochAbsent(mock) + mock.ExpectExec(epochSQLLoseRoot).WithArgs(int64(1), root, storeID).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(epochSQLLoseCarry).WithArgs(int64(1), root, storeID).WillReturnResult(sqlmock.NewResult(0, 0)) + if err := LoseVersionInTx(ctx, tx, storeID, root); err != nil { + t.Fatalf("LoseVersionInTx with no store_epoch row: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("LoseVersionInTx must write only the loss, at the default epoch 1: %v", err) + } + }) +} + +// TestStillServesAndResolveAnswerFromTheAddressRowAndTheEpochAlone pins the +// whole resolver. Each case registers exactly the statements the answer may +// use, the address's own row and then the epoch, so a third statement (for +// instance a look at the id's other rows, which is what the earlier chain +// check did) is an unexpected call the mock rejects. An address nobody minted, +// or another store's, is answered from the row alone: the epoch is not read. +func TestStillServesAndResolveAnswerFromTheAddressRowAndTheEpochAlone(t *testing.T) { + t.Parallel() + + const storeID = "status-store" + const currentEpoch = 5 + root := epochAddress(storeID, "record-a", 1) + + cases := []struct { + name string + row *epochTestRow + wantServes bool + wantRestriction EpochRestriction + readsEpoch bool + }{ + { + name: "a row minted in an earlier epoch and never lost is served", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1}, + wantServes: true, + wantRestriction: EpochRestrictionLive, + readsEpoch: true, + }, + { + name: "a row minted in the current epoch is served", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch}, + wantServes: true, + wantRestriction: EpochRestrictionLive, + readsEpoch: true, + }, + { + name: "a carry row is served on its own row, with no look at the root", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 3, carriedFrom: root}, + wantServes: true, + wantRestriction: EpochRestrictionLive, + readsEpoch: true, + }, + { + name: "a lost row is not served", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1, goneAtEpoch: 3}, + wantServes: false, + wantRestriction: EpochRestrictionGoneReorganization, + readsEpoch: true, + }, + { + name: "a row lost in the epoch it was minted in is not served", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch, goneAtEpoch: currentEpoch}, + wantServes: false, + wantRestriction: EpochRestrictionGoneReorganization, + readsEpoch: true, + }, + { + name: "a row minted in an epoch that has not happened yet fails closed", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch + 2}, + wantServes: false, + wantRestriction: EpochRestrictionGoneReorganization, + readsEpoch: true, + }, + { + name: "an address the store never minted is unknown", + row: nil, + wantServes: false, + wantRestriction: EpochRestrictionUnknown, + }, + { + name: "an address another store minted is unknown", + row: &epochTestRow{storeID: "other-store", mintedID: "record-a", mintedEpoch: 1}, + wantServes: false, + wantRestriction: EpochRestrictionUnknown, + }, + } + + for _, tc := range cases { + t.Run(tc.name+" (StillServes)", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, root, tc.row) + if tc.readsEpoch { + expectEpochRead(mock, currentEpoch) + } + + got, err := StillServesInTx(context.Background(), tx, storeID, root) + if err != nil { + t.Fatalf("StillServesInTx: %v", err) + } + if got != tc.wantServes { + t.Errorf("StillServesInTx = %v, want %v", got, tc.wantServes) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("StillServesInTx ran a different set of statements than the row read and the epoch read: %v", err) + } + }) + t.Run(tc.name+" (Resolve)", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, root, tc.row) + if tc.readsEpoch { + expectEpochRead(mock, currentEpoch) + } + + got, err := ResolveEpochInTx(context.Background(), tx, storeID, root) + if err != nil { + t.Fatalf("ResolveEpochInTx: %v", err) + } + if got.Restriction != tc.wantRestriction { + t.Errorf("ResolveEpochInTx.Restriction = %v, want %v", got.Restriction, tc.wantRestriction) + } + if got.ProducingStore != storeID { + t.Errorf("ResolveEpochInTx.ProducingStore = %q, want %q", got.ProducingStore, storeID) + } + var wantEpoch *int + if tc.readsEpoch { + wantEpoch = intPtr(currentEpoch) + } + switch { + case wantEpoch == nil && got.Epoch != nil: + t.Errorf("ResolveEpochInTx.Epoch = %d, want nil: an unknown address never looks the epoch up", *got.Epoch) + case wantEpoch != nil && got.Epoch == nil: + t.Errorf("ResolveEpochInTx.Epoch = nil, want the current epoch %d", *wantEpoch) + case wantEpoch != nil && *got.Epoch != *wantEpoch: + t.Errorf("ResolveEpochInTx.Epoch = %d, want the current epoch %d", *got.Epoch, *wantEpoch) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("ResolveEpochInTx ran a different set of statements than the row read and the epoch read: %v", err) + } + }) + } +} + +// TestCurrentAddressForInTxReadsAndNeverWrites pins that CurrentAddressFor is +// a pure read (bee-ghosttrack, review 5360880888, Major 1). The earlier form +// minted a fresh address under every read, which made a lost address look +// served again. Both Dolt legs open a read-write transaction for their read +// path and always roll it back, so a stray write would vanish with no error: +// the transaction type cannot enforce this. These cases register no ExpectExec, +// so any INSERT, UPDATE or DELETE is an unexpected call. +func TestCurrentAddressForInTxReadsAndNeverWrites(t *testing.T) { + t.Parallel() + + const storeID = "current-address-store" + const currentEpoch = 5 + root := epochAddress(storeID, "record-a", 1) + firstCarry := epochAddress(storeID, "record-a", 3) + newestCarry := epochAddress(storeID, "record-a", 5) + + cases := []struct { + name string + // asked is the address the caller holds; row is its row. + asked string + row *epochTestRow + // newest is the lineage's newest carry row, "" when it has none. + newest string + want string + }{ + { + name: "a survivor nothing carried returns its own address", + asked: root, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1}, + want: root, + }, + { + name: "the root of a carried lineage returns the newest carry", + asked: root, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1}, + newest: newestCarry, + want: newestCarry, + }, + { + name: "an older carry resolves through the lineage root, one lookup from any member", + asked: firstCarry, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 3, carriedFrom: root}, + newest: newestCarry, + want: newestCarry, + }, + { + name: "the newest carry returns itself", + asked: newestCarry, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 5, carriedFrom: root}, + newest: newestCarry, + want: newestCarry, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, tc.asked, tc.row) + expectEpochRead(mock, currentEpoch) + lookup := mock.ExpectQuery(epochSQLNewest).WithArgs(root, storeID, int64(currentEpoch)) + if tc.newest == "" { + lookup.WillReturnRows(sqlmock.NewRows([]string{"address"})) + } else { + lookup.WillReturnRows(sqlmock.NewRows([]string{"address"}).AddRow(tc.newest)) + } + + got, err := CurrentAddressForInTx(context.Background(), tx, storeID, tc.asked) + if err != nil { + t.Fatalf("CurrentAddressForInTx: %v", err) + } + if got != tc.want { + t.Errorf("CurrentAddressForInTx = %q, want %q", got, tc.want) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("CurrentAddressForInTx ran a different set of statements than reads: %v", err) + } + }) + } +} + +// TestCurrentAddressForInTxReturnsTypedErrorsForAddressesItCannotAnswer pins the +// two errors a caller can tell apart: an address the store never minted, and +// one it minted but no longer serves (lost, or from an epoch that has not +// happened). Neither runs the lineage lookup, and neither writes. +func TestCurrentAddressForInTxReturnsTypedErrorsForAddressesItCannotAnswer(t *testing.T) { + t.Parallel() + + const storeID = "typed-error-store" + const currentEpoch = 5 + address := epochAddress(storeID, "record-a", 1) + + cases := []struct { + name string + row *epochTestRow + readsEpoch bool + want error + notWant error + }{ + { + name: "a lost address is not served", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1, goneAtEpoch: 3}, + readsEpoch: true, + want: ErrEpochAddressNotServed, + notWant: ErrEpochAddressNotFound, + }, + { + name: "an address from an epoch that has not happened is not served", + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch + 1}, + readsEpoch: true, + want: ErrEpochAddressNotServed, + notWant: ErrEpochAddressNotFound, + }, + { + name: "an address the store never minted is not found", + row: nil, + want: ErrEpochAddressNotFound, + notWant: ErrEpochAddressNotServed, + }, + { + name: "an address another store minted is not found", + row: &epochTestRow{storeID: "other-store", mintedID: "record-a", mintedEpoch: 1}, + want: ErrEpochAddressNotFound, + notWant: ErrEpochAddressNotServed, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, address, tc.row) + if tc.readsEpoch { + expectEpochRead(mock, currentEpoch) + } + + got, err := CurrentAddressForInTx(context.Background(), tx, storeID, address) + if err == nil { + t.Fatalf("CurrentAddressForInTx = %q with no error, want %v", got, tc.want) + } + if !errors.Is(err, tc.want) { + t.Errorf("CurrentAddressForInTx error = %v, want it to wrap %v", err, tc.want) + } + if errors.Is(err, tc.notWant) { + t.Errorf("CurrentAddressForInTx error = %v, must not wrap %v: callers tell the two apart", err, tc.notWant) + } + if got != "" { + t.Errorf("CurrentAddressForInTx returned %q alongside an error, want an empty address", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + } +} + +// TestLoseVersionInTxMarksTheWholeLineageOnceAndOnlyWhereStillLive pins loss: +// it reads the address's row and the epoch (never creating the singleton), then +// marks the lineage ROOT's own row and every carry row of that root, each +// statement guarded by gone_at_epoch IS NULL so that nothing already lost is +// touched and nothing is ever cleared. Losing a lineage that is already lost +// matches no rows and is not an error, so a retried transaction may replay it. +func TestLoseVersionInTxMarksTheWholeLineageOnceAndOnlyWhereStillLive(t *testing.T) { + t.Parallel() + + const storeID = "lose-store" + const currentEpoch = 4 + root := epochAddress(storeID, "record-a", 1) + carry := epochAddress(storeID, "record-a", 3) + + cases := []struct { + name string + asked string + row *epochTestRow + rootAffected int64 + carryRows int64 + }{ + { + name: "losing the root marks the root row and its carry rows", + asked: root, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1}, + rootAffected: 1, + carryRows: 1, + }, + { + name: "losing a carry address marks the same lineage through its root", + asked: carry, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 3, carriedFrom: root}, + rootAffected: 1, + carryRows: 1, + }, + { + name: "a lineage that was never carried still runs both guarded statements", + asked: root, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1}, + rootAffected: 1, + carryRows: 0, + }, + { + name: "a lineage that is already lost matches no rows and is not an error", + asked: root, + row: &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1, goneAtEpoch: 3}, + rootAffected: 0, + carryRows: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, tc.asked, tc.row) + expectEpochRead(mock, currentEpoch) + mock.ExpectExec(epochSQLLoseRoot). + WithArgs(int64(currentEpoch), root, storeID). + WillReturnResult(sqlmock.NewResult(0, tc.rootAffected)) + mock.ExpectExec(epochSQLLoseCarry). + WithArgs(int64(currentEpoch), root, storeID). + WillReturnResult(sqlmock.NewResult(0, tc.carryRows)) + + if err := LoseVersionInTx(context.Background(), tx, storeID, tc.asked); err != nil { + t.Fatalf("LoseVersionInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("LoseVersionInTx did not run the root update and then the carry update, each guarded: %v", err) + } + }) + } +} + +// TestLoseVersionInTxReportsAnAddressTheStoreNeverMinted pins that a loss +// naming an address this store never minted is an error and writes nothing. +func TestLoseVersionInTxReportsAnAddressTheStoreNeverMinted(t *testing.T) { + t.Parallel() + + const storeID = "lose-absent-store" + address := epochAddress(storeID, "record-a", 1) + + cases := []struct { + name string + row *epochTestRow + }{ + {name: "an address nobody minted", row: nil}, + {name: "an address another store minted", row: &epochTestRow{storeID: "other-store", mintedID: "record-a", mintedEpoch: 1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, address, tc.row) + + err := LoseVersionInTx(context.Background(), tx, storeID, address) + if !errors.Is(err, ErrEpochAddressNotFound) { + t.Fatalf("LoseVersionInTx = %v, want it to wrap ErrEpochAddressNotFound", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("a loss that cannot be recorded must run the row read and nothing else: %v", err) + } + }) + } +} + +// TestLoseVersionInTxSurfacesAStatementFailure pins that a failed update is +// returned, not swallowed. A lineage member minted in an epoch that has not +// happened yet violates the table's gone-not-before-mint check, and the loss +// must report that anomaly rather than paper over it. +func TestLoseVersionInTxSurfacesAStatementFailure(t *testing.T) { + t.Parallel() + + const storeID = "lose-failure-store" + address := epochAddress(storeID, "record-a", 1) + checkViolation := errors.New("check constraint violated") + + _, mock, tx := beginMockTx(t) + expectEpochRow(mock, address, &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: 1}) + expectEpochRead(mock, 2) + mock.ExpectExec(epochSQLLoseRoot). + WithArgs(int64(2), address, storeID). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(epochSQLLoseCarry). + WithArgs(int64(2), address, storeID). + WillReturnError(checkViolation) + + err := LoseVersionInTx(context.Background(), tx, storeID, address) + if !errors.Is(err, checkViolation) { + t.Fatalf("LoseVersionInTx = %v, want it to wrap the failed statement's error", err) + } +} + +// TestBumpEpochInTxWritesOnlyStoreEpoch pins that a plain bump (restore and +// destructive reinit) touches the counter and nothing else. Restore replaces +// the database and reinit keeps what survives, so neither may write an address +// row: an address row a bump wrote would be an address that was not minted. +// Only BumpEpochCarryingInTx, for a token-scheme change, writes address rows. +func TestBumpEpochInTxWritesOnlyStoreEpoch(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, 3) + mock.ExpectExec(`UPDATE store_epoch SET epoch = epoch \+ 1, bumped_at = \?, bumped_reason = \? WHERE id = 1`). + WithArgs(sqlmock.AnyArg(), "restore"). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectEpochRead(mock, 4) + + got, err := BumpEpochInTx(context.Background(), tx, "bump-store", "restore") + if err != nil { + t.Fatalf("BumpEpochInTx: %v", err) + } + if got != 4 { + t.Errorf("BumpEpochInTx = %d, want 4", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("BumpEpochInTx ran statements beyond the store_epoch read, update and read-back: %v", err) + } +} + +// TestABumpUnderOneStoreIDMovesTheEpochSeenUnderEveryOther pins the sentence +// the epoch functions' docs now carry (bee-ghosttrack, review 5360880888, +// Minor 3): the counter is per database, not per store, and storeID appears +// only in error text. No SQL here takes the storeID as an argument, so a bump +// made under one storeID is seen under every storeID that shares the database. +// There is deliberately no storeID guard: the conformance suite uses several +// synthetic storeIDs in one database. +func TestABumpUnderOneStoreIDMovesTheEpochSeenUnderEveryOther(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, 2) + mock.ExpectExec(`UPDATE store_epoch SET epoch = epoch \+ 1, bumped_at = \?, bumped_reason = \? WHERE id = 1`). + WithArgs(sqlmock.AnyArg(), "restore"). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectEpochRead(mock, 3) + expectEpochRead(mock, 3) + + bumped, err := BumpEpochInTx(context.Background(), tx, "store-one", "restore") + if err != nil { + t.Fatalf("BumpEpochInTx under store-one: %v", err) + } + seen, err := CurrentEpochInTx(context.Background(), tx, "store-two") + if err != nil { + t.Fatalf("CurrentEpochInTx under store-two: %v", err) + } + if seen != bumped { + t.Errorf("CurrentEpochInTx(store-two) = %d after a bump under store-one reported %d, want the same epoch: the counter is per database", seen, bumped) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} + +// TestMintUnderEpochInTxNeverHandsBackAnAddressTheStoreDoesNotServe pins the +// mint's refusal. A mint is deterministic in (storeID, id, epoch), so it can +// find a row already sitting at the address it computes; that row is only +// possible when the lineage was minted in this epoch and lost in this same +// epoch. Handing that address back would give the caller an address that +// reads Gone, so the mint returns ErrEpochAddressNotServed instead, and it +// does not touch the row. +func TestMintUnderEpochInTxNeverHandsBackAnAddressTheStoreDoesNotServe(t *testing.T) { + t.Parallel() + + const storeID = "mint-store" + const currentEpoch = 5 + address := epochAddress(storeID, "record-a", currentEpoch) + + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, currentEpoch) + expectEpochRow(mock, address, &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch, goneAtEpoch: currentEpoch}) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if !errors.Is(err, ErrEpochAddressNotServed) { + t.Fatalf("MintUnderEpochInTx = (%q, %v), want it to wrap ErrEpochAddressNotServed", got, err) + } + if got != "" { + t.Errorf("MintUnderEpochInTx returned %q alongside the refusal, want an empty address", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("a refused mint must read and write nothing else: %v", err) + } +} + +// TestMintUnderEpochInTxInsertsOnlyAFreshRootRow pins that the only write a +// mint ever makes is the INSERT of its own root row. A mint that finds its +// address already served returns it with no write at all, so it can never +// modify, retire or revive another row. +func TestMintUnderEpochInTxInsertsOnlyAFreshRootRow(t *testing.T) { + t.Parallel() + + const storeID = "mint-store" + const currentEpoch = 5 + address := epochAddress(storeID, "record-a", currentEpoch) + + t.Run("an absent row is inserted as a root row", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, currentEpoch) + expectEpochRow(mock, address, nil) + mock.ExpectExec(epochSQLInsertRoot). + WithArgs(address, storeID, "record-a", int64(currentEpoch), sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if err != nil { + t.Fatalf("MintUnderEpochInTx: %v", err) + } + if got != address { + t.Errorf("MintUnderEpochInTx = %q, want %q", got, address) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the mint's only write must be the root INSERT: %v", err) + } + }) + + t.Run("a served row at the address is returned with no write", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, currentEpoch) + expectEpochRow(mock, address, &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch}) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if err != nil { + t.Fatalf("MintUnderEpochInTx: %v", err) + } + if got != address { + t.Errorf("MintUnderEpochInTx = %q, want %q", got, address) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("minting an address that is already served must write nothing: %v", err) + } + }) +} + +// TestMintUnderEpochInTxTrustsADuplicateKeyOnlyAsFarAsItCanReadTheRow pins +// what a mint does when its INSERT reports a duplicate key for an address its +// own read found absent: another transaction inserted the row, and may have +// lost it since. The mint reads the row again and returns the address only +// when that row is served; it refuses a row that is not served, and a row it +// cannot read at all is a failure, never the success item B3 once assumed. +func TestMintUnderEpochInTxTrustsADuplicateKeyOnlyAsFarAsItCanReadTheRow(t *testing.T) { + t.Parallel() + + const storeID = "mint-race-store" + const currentEpoch = 5 + address := epochAddress(storeID, "record-a", currentEpoch) + duplicate := errors.New("Error 1062: Duplicate entry for key 'PRIMARY'") + + // expectLostInsert registers a mint whose read finds no row and whose + // INSERT then hits the row another transaction wrote. + expectLostInsert := func(mock sqlmock.Sqlmock) { + expectEpochRead(mock, currentEpoch) + expectEpochRow(mock, address, nil) + mock.ExpectExec(epochSQLInsertRoot). + WithArgs(address, storeID, "record-a", int64(currentEpoch), sqlmock.AnyArg()). + WillReturnError(duplicate) + } + + t.Run("a served row is the one the mint would have inserted", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectLostInsert(mock) + expectEpochRow(mock, address, &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch}) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if err != nil { + t.Fatalf("MintUnderEpochInTx after losing the insert to a row it then reads served: %v", err) + } + if got != address { + t.Errorf("MintUnderEpochInTx = %q, want %q", got, address) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("a row lost since it was inserted is refused", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectLostInsert(mock) + expectEpochRow(mock, address, &epochTestRow{storeID: storeID, mintedID: "record-a", mintedEpoch: currentEpoch, goneAtEpoch: currentEpoch}) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if !errors.Is(err, ErrEpochAddressNotServed) { + t.Fatalf("MintUnderEpochInTx = (%q, %v), want it to wrap ErrEpochAddressNotServed: the row at the address reads Gone", got, err) + } + if got != "" { + t.Errorf("MintUnderEpochInTx returned %q alongside the refusal, want an empty address", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("a row the transaction cannot read is not success", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectLostInsert(mock) + expectEpochRow(mock, address, nil) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if err == nil { + t.Fatalf("MintUnderEpochInTx = %q, nil, want an error: nothing this transaction can read says the store serves the row another transaction inserted", got) + } + if !errors.Is(err, duplicate) { + t.Errorf("MintUnderEpochInTx error = %v, want it to wrap the duplicate-key error, so a caller can tell a race from a refusal", err) + } + if errors.Is(err, ErrEpochAddressNotServed) { + t.Errorf("MintUnderEpochInTx error = %v wraps ErrEpochAddressNotServed, want not: the row's status is unknown, not Gone", err) + } + if got != "" { + t.Errorf("MintUnderEpochInTx returned %q alongside the error, want an empty address", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("a failed re-read reports both errors", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + expectLostInsert(mock) + readFailure := errors.New("read failed") + mock.ExpectQuery(epochSQLRow).WithArgs(address).WillReturnError(readFailure) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, "record-a") + if !errors.Is(err, duplicate) || !errors.Is(err, readFailure) { + t.Fatalf("MintUnderEpochInTx = (%q, %v), want an error wrapping both the duplicate key and the failed re-read", got, err) + } + if got != "" { + t.Errorf("MintUnderEpochInTx returned %q alongside the error, want an empty address", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) +} + +// TestMintUnderEpochInTxRefusesAPairWhoseAddressCouldOutgrowItsColumn pins the +// bound on epoch_minted_addresses.address, a VARCHAR(255) PRIMARY KEY: a pair +// one byte too long is refused with ErrEpochAddressTooLong before any +// statement runs, and the widest pair that fits is minted. The bound is the +// address at the widest epoch the INT counter holds, so a pair that mints +// today still fits after any number of bumps. +func TestMintUnderEpochInTxRefusesAPairWhoseAddressCouldOutgrowItsColumn(t *testing.T) { + t.Parallel() + + storeID := strings.Repeat("s", 115) + widest := strings.Repeat("i", 115) + if n := len(epochAddress(storeID, widest, math.MaxInt32)); n != maxEpochAddressLen { + t.Fatalf("the widest pair composes to %d bytes at the widest epoch, want exactly %d: the boundary below is not at the limit", n, maxEpochAddressLen) + } + + t.Run("one byte over is refused before any statement", func(t *testing.T) { + t.Parallel() + _, mock, tx := beginMockTx(t) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, widest+"i") + if !errors.Is(err, ErrEpochAddressTooLong) { + t.Fatalf("MintUnderEpochInTx(one byte over) = (%q, %v), want it to wrap ErrEpochAddressTooLong", got, err) + } + if got != "" { + t.Errorf("MintUnderEpochInTx returned %q alongside the refusal, want an empty address", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("a refused pair must run no statement: %v", err) + } + }) + + t.Run("the widest pair that fits is minted", func(t *testing.T) { + t.Parallel() + // Epoch 1 is the epoch whose address is SHORTEST, so this pair passes + // only because the bound is charged at the widest epoch and met there. + address := epochAddress(storeID, widest, 1) + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, 1) + expectEpochRow(mock, address, nil) + mock.ExpectExec(epochSQLInsertRoot). + WithArgs(address, storeID, widest, int64(1), sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + + got, err := MintUnderEpochInTx(context.Background(), tx, storeID, widest) + if err != nil { + t.Fatalf("MintUnderEpochInTx(the widest pair): %v", err) + } + if got != address { + t.Errorf("MintUnderEpochInTx = %q, want %q", got, address) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) +} + +// TestBumpEpochCarryingInTxCarriesTheWidestMintablePairToTheWidestEpoch is the +// carry half of the address bound. A token-scheme change re-encodes every live +// pair at the new epoch, so the widest pair a mint accepts, carried to the +// widest epoch the counter holds, must produce an address the column holds; +// the mint's bound is what guarantees it. +func TestBumpEpochCarryingInTxCarriesTheWidestMintablePairToTheWidestEpoch(t *testing.T) { + t.Parallel() + + storeID := strings.Repeat("s", 115) + widest := strings.Repeat("i", 115) + if err := validateEpochAddressInputs(storeID, widest); err != nil { + t.Fatalf("validateEpochAddressInputs(the widest pair) = %v, want nil: a mint accepts it", err) + } + root := epochAddress(storeID, widest, 1) + carried := epochAddress(storeID, widest, math.MaxInt32) + if len(carried) > maxEpochAddressLen { + t.Fatalf("the widest mintable pair carried to epoch %d is %d bytes, over the %d the column holds", math.MaxInt32, len(carried), maxEpochAddressLen) + } + + _, mock, tx := beginMockTx(t) + expectEpochRead(mock, math.MaxInt32-1) + mock.ExpectExec(`UPDATE store_epoch SET epoch = epoch \+ 1, bumped_at = \?, bumped_reason = \? WHERE id = 1`). + WithArgs(sqlmock.AnyArg(), "token-scheme-change"). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectEpochRead(mock, math.MaxInt32) + mock.ExpectQuery(`SELECT address, store_id, minted_id, carried_from FROM epoch_minted_addresses WHERE gone_at_epoch IS NULL AND minted_epoch < \? ORDER BY store_id, minted_id, minted_epoch DESC`). + WithArgs(int64(math.MaxInt32)). + WillReturnRows(sqlmock.NewRows([]string{"address", "store_id", "minted_id", "carried_from"}).AddRow(root, storeID, widest, nil)) + mock.ExpectExec(`INSERT INTO epoch_minted_addresses \(address, store_id, minted_id, minted_epoch, minted_at, carried_from\) VALUES \(\?, \?, \?, \?, \?, \?\)`). + WithArgs(carried, storeID, widest, int64(math.MaxInt32), sqlmock.AnyArg(), root). + WillReturnResult(sqlmock.NewResult(0, 1)) + + got, err := BumpEpochCarryingInTx(context.Background(), tx, storeID, "token-scheme-change") + if err != nil { + t.Fatalf("BumpEpochCarryingInTx to the widest epoch: %v", err) + } + if got != math.MaxInt32 { + t.Errorf("BumpEpochCarryingInTx = %d, want %d", got, math.MaxInt32) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the carry must insert exactly the %d-byte address of the widest pair at the widest epoch: %v", len(carried), err) + } +} + +// TestEpochAddressNeverCollidesAcrossDifferentStoreIDBoundaries is the +// regression test for gastownhall/beads#6664 (bee-ghosttrack, maintainer +// review 5268699223, item B2, first part): epochAddress's ":"-joined +// encoding is ambiguous — a colon inside storeID or id lets two +// semantically different (storeID, id, epoch) triples collide on the +// identical address string. This directly contradicts the package doc's +// "persisted once at mint and never rewritten in place" promise: a +// colliding second mint would silently steal the first triple's row +// (upsertEpochMintedAddressInTx's exists-branch UPDATE has no store_id/ +// minted_id guard). +func TestEpochAddressNeverCollidesAcrossDifferentStoreIDBoundaries(t *testing.T) { + t.Parallel() + + // Under the old "epch:%s:%s:%d" encoding, these two distinct triples + // both produce "epch:store-a:b:c:1": the colon inside the first + // triple's id ("b:c") lands exactly on the second triple's + // storeID/id boundary ("store-a:b" / "c"). + const storeA, idA = "store-a", "b:c" + const storeB, idB = "store-a:b", "c" + a := epochAddress(storeA, idA, 1) + b := epochAddress(storeB, idB, 1) + if a == b { + t.Fatalf("epochAddress(%q, %q, 1) and epochAddress(%q, %q, 1) both produced %q; a colon inside storeID or id must not let two different triples collide on the same address (B2, gastownhall/beads#6664 review 5268699223)", storeA, idA, storeB, idB, a) + } +} + +// TestUpsertEpochMintedAddressInTxIsANoOpWhenTheAddressAlreadyExists is the +// regression test for gastownhall/beads#6664 (bee-ghosttrack, maintainer +// review 5268699223, item B2, third part): once epochAddress is injective +// (TestEpochAddressNeverCollidesAcrossDifferentStoreIDBoundaries above), a +// row's address already existing provably means its stored (store_id, +// minted_id, minted_epoch) triple already matches the incoming one exactly +// — so the exists branch must perform no write at all, rather than the +// UPDATE it runs today, which (before epochAddress is injective) can steal +// another store's row through a colliding address. +func TestUpsertEpochMintedAddressInTxIsANoOpWhenTheAddressAlreadyExists(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + address := epochAddress("no-op-store", "record-a", 1) + + // Deliberately no ExpectExec/ExpectQuery at all: any write this branch + // attempts is an unexpected call sqlmock rejects. + err := upsertEpochMintedAddressInTx(context.Background(), tx, address, "no-op-store", "record-a", 1, true) + if err != nil { + t.Fatalf("upsertEpochMintedAddressInTx(exists=true) = %v, want nil: an already-existing address must be a no-op, not attempt a write this test set up no expectation for (B2, gastownhall/beads#6664 review 5268699223)", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet no-op-on-exists SQL expectations: %v", err) + } +} + +// TestEnsureStoreEpochRowSeedsTheSingletonWithInsertIgnore is the regression +// test for the "Unverified" note of gastownhall/beads#6664 (bee-ghosttrack, +// review 5360880888): ensureStoreEpochRow seeded +// store_epoch's singleton with a plain INSERT where version_history.go seeds +// the very same row with INSERT IGNORE. It follows that seed's shape: read +// first, seed only when the row is absent (so the seed happens once per store, +// not once per mint), then read the row back so the epoch it reports is what +// the row actually holds. +func TestEnsureStoreEpochRowSeedsTheSingletonWithInsertIgnore(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + + // store_epoch starts empty (migration 0067), so the first read finds no row. + mock.ExpectQuery(`SELECT epoch FROM store_epoch WHERE id = 1`). + WillReturnRows(sqlmock.NewRows([]string{"epoch"})) + mock.ExpectExec(`INSERT IGNORE INTO store_epoch \(id, epoch\) VALUES \(1, 1\)`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT epoch FROM store_epoch WHERE id = 1`). + WillReturnRows(sqlmock.NewRows([]string{"epoch"}).AddRow(1)) + + got, err := ensureStoreEpochRow(context.Background(), tx) + if err != nil { + t.Fatalf("ensureStoreEpochRow on an empty store_epoch: %v", err) + } + if got != 1 { + t.Fatalf("ensureStoreEpochRow on an empty store_epoch = epoch %d, want 1 (a freshly seeded singleton starts at epoch 1)", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet store_epoch seed SQL expectations: %v", err) + } +} + +// TestEnsureStoreEpochRowReportsTheEpochOfARowItsSeedDidNotWrite pins why the +// seed reads back instead of returning the literal 1 it would have written: +// INSERT IGNORE writes nothing when the singleton already exists, and the row +// it leaves behind may hold a later epoch than the one this call tried to seed. +func TestEnsureStoreEpochRowReportsTheEpochOfARowItsSeedDidNotWrite(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + + mock.ExpectQuery(`SELECT epoch FROM store_epoch WHERE id = 1`). + WillReturnRows(sqlmock.NewRows([]string{"epoch"})) + // The seed is ignored: another writer's row is already there. + mock.ExpectExec(`INSERT IGNORE INTO store_epoch \(id, epoch\) VALUES \(1, 1\)`). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(`SELECT epoch FROM store_epoch WHERE id = 1`). + WillReturnRows(sqlmock.NewRows([]string{"epoch"}).AddRow(3)) + + got, err := ensureStoreEpochRow(context.Background(), tx) + if err != nil { + t.Fatalf("ensureStoreEpochRow when the seed is ignored: %v", err) + } + if got != 3 { + t.Fatalf("ensureStoreEpochRow when the seed is ignored = epoch %d, want 3 (the epoch the surviving row holds, not the 1 the ignored seed would have written)", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet ignored-seed SQL expectations: %v", err) + } +} + +// handWrittenEpochAddresses returns the position of every string literal in src +// that writes an address by hand, meaning one that starts with the address +// encoding's prefix. It reads string literals only, so a comment that merely +// describes an address, or a call to epochAddress, is never reported. +func handWrittenEpochAddresses(filename string, src any) ([]string, error) { + fset := token.NewFileSet() + parsed, err := parser.ParseFile(fset, filename, src, 0) + if err != nil { + return nil, err + } + // The prefix is assembled here so this guard's own source holds no + // literal that starts with it. + addressPrefix := "epch" + ":" + var hits []string + ast.Inspect(parsed, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + value, err := strconv.Unquote(lit.Value) + if err != nil { + return true + } + if strings.HasPrefix(value, addressPrefix) { + hits = append(hits, fset.Position(lit.Pos()).String()+": "+lit.Value) + } + return true + }) + return hits, nil +} + +// TestEpochCASTestsBuildEveryAddressWithEpochAddress pins the rule that this +// file never writes an address by hand (bee-ghosttrack, review 5360880888, +// Minor 4). The review found colon-form literals of the encoding that +// epochAddress retired, and one length-prefixed literal whose lengths were +// wrong. A hand-written address goes stale the moment the encoding moves, +// while a test that builds it with epochAddress cannot disagree with the code +// under test. +// +// The server leg's publication tests are held to the same rule (#6664 review): +// their fake serves a row at any address, so a stale one would still pass +// there. They cannot call epochAddress from that package, so they mint the +// address instead. +func TestEpochCASTestsBuildEveryAddressWithEpochAddress(t *testing.T) { + t.Parallel() + + for _, filename := range []string{ + "epoch_cas_test.go", + filepath.Join("..", "dolt", "epoch_cas_publish_test.go"), + } { + hits, err := handWrittenEpochAddresses(filename, nil) + if err != nil { + t.Fatalf("parse %s: %v", filename, err) + } + for _, hit := range hits { + t.Errorf("%s writes an address by hand; build it with epochAddress(storeID, id, epoch), or mint it where epochAddress is out of reach", hit) + } + } +} + +// TestHandWrittenAddressGuardHasTeeth proves the guard above can fail: it must +// report a colon-form literal and a length-prefixed literal, and must stay +// quiet for a call to epochAddress and for a comment describing an address. +func TestHandWrittenAddressGuardHasTeeth(t *testing.T) { + t.Parallel() + + prefix := "epch" + ":" + cases := []struct { + name string + src string + want int + }{ + {name: "a colon-form literal", src: "package p\nvar a = \"" + prefix + "store:id:1\"\n", want: 1}, + {name: "a length-prefixed literal", src: "package p\nvar a = \"" + prefix + "5:store:2:id:1\"\n", want: 1}, + {name: "two literals", src: "package p\nvar a, b = \"" + prefix + "x\", \"" + prefix + "y\"\n", want: 2}, + {name: "a call to epochAddress", src: "package p\nvar a = epochAddress(\"store\", \"id\", 1)\n", want: 0}, + {name: "a comment describing an address", src: "package p\n// the old form was " + prefix + "store:id:1\nvar a = 1\n", want: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + hits, err := handWrittenEpochAddresses("sample.go", tc.src) + if err != nil { + t.Fatalf("parse sample: %v", err) + } + if len(hits) != tc.want { + t.Errorf("the guard reported %d hand-written addresses (%v), want %d", len(hits), hits, tc.want) + } + }) + } +} diff --git a/internal/storage/leg_contract_wiring_test.go b/internal/storage/leg_contract_wiring_test.go index 00fdcdd46..960efc16b 100644 --- a/internal/storage/leg_contract_wiring_test.go +++ b/internal/storage/leg_contract_wiring_test.go @@ -106,8 +106,10 @@ func init() { "RunImporterWiresTheCrossPlaneEdgeBetweenItsRows": importerOneAccessorWaiverReason, "RunImporterReportsTheCycleEdgeItDropped": importerOneAccessorWaiverReason, + "RunACarriedAddressKeepsResolvingAcrossLaterBumps": retentionEpochNoRoleWaiverReason, "RunADestructiveOperationEnumeratesAffectedAddressesFirst": retentionEpochNoRoleWaiverReason, "RunAHoldPreventsRemovalAndReportsInRetainedBounds": retentionEpochNoRoleWaiverReason, + "RunALostAddressStaysGoneAcrossASchemeChange": retentionEpochNoRoleWaiverReason, "RunAStoreThatRemovesStateStillAnswersGoneDurably": retentionEpochNoRoleWaiverReason, "RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone": retentionEpochNoRoleWaiverReason, "RunAnAddressNeverResolvesToADifferentState": retentionEpochNoRoleWaiverReason, @@ -121,6 +123,7 @@ func init() { "RunRemovalNeverReassignsASurvivingAddress": retentionEpochNoRoleWaiverReason, "RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly": retentionEpochNoRoleWaiverReason, "RunRemovalReportsTheSurvivingRetainedWindow": retentionEpochNoRoleWaiverReason, + "RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose": retentionEpochNoRoleWaiverReason, "RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation": crossRecordInvariantNoRoleWaiverReason, "RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards": crossRecordInvariantNoRoleWaiverReason, @@ -185,30 +188,44 @@ const cycleTracksWireGapWaiverReason = "issueops.DetectCyclesRequest.IncludeTrac "\"L-cycles-tracks\" — rather than silently answering the narrower walk); no slice through S13 adds a wire " + "parameter for it, so closing this is a new, not-yet-numbered slice rather than a missing test line" -// retentionEpochNoRoleWaiverReason covers the fifteen RetentionFixture and -// EpochFixture cases: PERMANENTLY unservable over HTTP, not merely unwired -// yet. +// retentionEpochNoRoleWaiverReason covers the eighteen RetentionFixture and +// EpochFixture cases (backend/conformance/retention_epoch_contract.go). +// All but four of the participation-generation cases run on this leg because +// their subject is a create or an update the client can send +// (internal/httpclient/served_participation_generation_test.go; those four +// stay waived under participationGenerationWireGapWaiverReason). These +// cases are different: every hook of these two fixtures IS the operation +// under test, and none has a form on the wire: +// - EpochFixture's hooks (CurrentEpoch, BumpEpoch, MintUnderEpoch, +// StillServes, Resolve, CurrentAddressFor, LoseVersion) are the R20 epoch +// operations themselves: methods on the concrete *DoltStore and +// *EmbeddedDoltStore over issueops' *InTx bodies, on no storage +// interface, no publicops role and no wire operation. Every call path +// into them starts in a test: the methods are called only from each +// store leg's own _test.go files (its EpochFixture, which the +// backend/conformance contracts drive, and its epoch tests), and the +// *InTx bodies only from those methods, from each other and from tests +// (the uow leg's fixture, through a test-only type assertion, and +// issueops' and the dolt leg's own tests). No command, role or server +// handler reaches them, so no client mutation touches epoch state either. +// - RetentionFixture's hooks (Resolve, Remove, Hold, ForceRemove, Erase, +// Mint) are retention operations no backend implements yet: every +// store leg wires them nil and its cases skip, and the wire has no +// removal, hold, erasure or gone/unknown resolve to bind them to. // -// Both fixtures are RAW STORAGE HOOKS, not a publicops role: RetentionFixture -// and EpochFixture (backend/conformance/retention_epoch_contract.go) carry no -// publicops.XXX field at all, only closures like Resolve, Remove, Hold, -// Erase, Mint, CurrentEpoch, and BumpEpoch that reach past every role -// straight at a backend's own address/epoch/retention bookkeeping — the same -// bookkeeping a storage engine keeps about itself, not a capability it was -// ever going to publish to a caller. dolt, embeddeddolt, and uow all wire -// this contract because all three ARE storage engines with that bookkeeping -// to show; the http leg is a REMOTE CLIENT of one, with no address table, -// epoch counter, or retention window of its own to report — there is nothing -// behind this door for a wire operation to open onto, now or in a later -// slice, unless some future design gives retention/epoch state its own -// publicops role and wire surface. That would be a new role, not a missing -// test line, which is why this is a named waiver rather than a ceiling -// entry. -const retentionEpochNoRoleWaiverReason = "RetentionFixture and EpochFixture carry raw address/epoch/retention " + - "storage hooks (Resolve, Remove, Hold, Erase, Mint, CurrentEpoch, BumpEpoch, ...) with no publicops role " + - "behind them; dolt, embeddeddolt and uow wire this contract because they ARE the storage engine this " + - "bookkeeping belongs to, and the http leg is a remote client with no such bookkeeping of its own to report " + - "— there is no role or wire surface for this to ever route through unless one is designed from scratch" +// Binding the hooks to the reference store behind the served env would run +// the epoch cases green, but as a property of that store, never of the +// client; the store legs already pin it. The waiver stands until retention or +// epoch state gets a publicops role and a wire operation of its own, which +// would be a new role, not a missing test line. +const retentionEpochNoRoleWaiverReason = "every RetentionFixture/EpochFixture hook is the operation under test and " + + "none has a form on the http wire: the R20 epoch operations (CurrentEpoch, BumpEpoch, MintUnderEpoch, " + + "StillServes, Resolve, CurrentAddressFor, LoseVersion) are concrete-store methods on no storage interface, " + + "publicops role or wire operation, reached only from tests (each store leg's fixture and epoch tests call the " + + "methods, the uow fixture calls the issueops bodies through a test-only type assertion), and the retention " + + "hooks (Resolve, Remove, Hold, ForceRemove, Erase, Mint) are implemented by no backend and have no wire " + + "removal, hold, erasure or resolve; binding them to the served env's reference store would test that store, " + + "not the client" // crossRecordInvariantNoRoleWaiverReason covers the seven CrossRecordInvariantFixture // cases, for the same shape of reason as retentionEpochNoRoleWaiverReason. diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index f81aca9bc..68e8b9b75 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -121,6 +121,7 @@ go_library( "migrations/ignored/0028_add_wisps_participation_generation.up.sql", "migrations/0070_add_participation_generation.up.sql", "migrations/0071_add_removed_restriction.up.sql", + "migrations/0072_add_epoch_minted_addresses.up.sql", ], importpath = "github.com/steveyegge/beads/internal/storage/schema", visibility = ["//:__subpackages__"], @@ -156,6 +157,7 @@ go_test( "migration_0069_datetime_precision_test.go", "migration_0070_add_participation_generation_test.go", "migration_0071_add_removed_restriction_test.go", + "migration_0072_add_epoch_minted_addresses_test.go", "migration_body_drain_test.go", "migration_freeze_test.go", "migration_watchdog_test.go", diff --git a/internal/storage/schema/migration_0071_add_removed_restriction_test.go b/internal/storage/schema/migration_0071_add_removed_restriction_test.go index 2b4da39ac..5ca6598d5 100644 --- a/internal/storage/schema/migration_0071_add_removed_restriction_test.go +++ b/internal/storage/schema/migration_0071_add_removed_restriction_test.go @@ -22,19 +22,11 @@ import ( const migration0071Up = "0071_add_removed_restriction.up.sql" const migration0071Down = "0071_add_removed_restriction.down.sql" -// TestLatestVersionIncludesMigration0071 pins the real next free slot this -// phase claims, superseding 0070's own version of this test -// (migration_0070_add_participation_generation_test.go). Deliberately a -// hardcoded literal for the same reason 0067's, 0068's, 0069's and 0070's -// were: LatestVersion() drifting to 71 for the wrong reason should still be -// caught by this test failing to explain why 71 is removed-restriction-shaped, -// which the CLI test below checks. -func TestLatestVersionIncludesMigration0071(t *testing.T) { - const want = 71 - if got := LatestVersion(); got != want { - t.Fatalf("LatestVersion() = %d, want %d (the migration slot claimed for issue_versions.removed_restriction)", got, want) - } -} +// TestLatestVersionIncludesMigration0071 (pinning LatestVersion() == 71) is +// superseded by TestLatestVersionIncludesMigration0072 +// (migration_0072_add_epoch_minted_addresses_test.go) now that 0072 claims +// the next free slot -- only one such pin lives at a time, matching how this +// test itself already superseded 0070's own version. // TestMigration0071AddsRemovedRestriction is a pure-Go, DB-independent check // of the frozen migration bytes themselves — it runs even where no `dolt` diff --git a/internal/storage/schema/migration_0072_add_epoch_minted_addresses_test.go b/internal/storage/schema/migration_0072_add_epoch_minted_addresses_test.go new file mode 100644 index 000000000..67389a0e0 --- /dev/null +++ b/internal/storage/schema/migration_0072_add_epoch_minted_addresses_test.go @@ -0,0 +1,184 @@ +package schema + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// R20's epoch-transition enforcement (gastownhall/beads#5898 revision 9, +// #6136) adds one brand-new table, +// epoch_minted_addresses -- see +// migrations/0072_add_epoch_minted_addresses.up.sql for the full rationale. +// This migration is a plain, unguarded +// CREATE TABLE IF NOT EXISTS with no ALTER and no PREPARE, so it needs none +// of 0067/0068's CLI-hazard-override tests (already covered, for every +// migration including this one, by TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified +// and TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities). + +// TestLatestVersionIncludesMigration0072 pins the real next free slot this +// phase claims, superseding 0071's own version of this test +// (migration_0071_add_removed_restriction_test.go; only one such pin lives at +// a time, the same way 0071's superseded 0070's). +// +// This slice originally claimed 0069 too. It was renumbered because +// R7.1's removed_restriction migration -- a PARALLEL SIBLING of this branch, +// not an ancestor -- had also claimed 0069, and the two are now linearized: +// R7.1 goes first (it is published beneath gastownhall/beads#6661), this one +// takes the next contiguous slot. Both then moved up one more slot when +// upstream's 0069_widen_issue_versions_datetime_precision +// (gastownhall/beads#6675) landed first, and once more when +// gastownhall/beads#6943's 0070_add_participation_generation (the +// versioned-history write fence) was ordered ahead of them: migrations are +// numbered by landing order, so R7.1 is 0071 and this is 0072. +// Two files sharing a +// version number panic checkNoDuplicateVersions at store open, before any +// command's RunE, which bricks every bd invocation rather than merely +// reddening a package -- see the same hazard on #6358 vs #6008. +// +// Deliberately a hardcoded literal for the same reason 0070's and 0071's +// were: LatestVersion() drifting to 72 for the wrong reason (an unrelated +// migration landing first) should still be caught by this test failing to +// explain why 72 is epoch-minted-addresses-shaped. +func TestLatestVersionIncludesMigration0072(t *testing.T) { + const want = 72 + if got := LatestVersion(); got != want { + t.Fatalf("LatestVersion() = %d, want %d (epoch_minted_addresses migration slot claimed by gastownhall/beads#6664)", got, want) + } +} + +const migration0072Up = "0072_add_epoch_minted_addresses.up.sql" +const migration0072Down = "0072_add_epoch_minted_addresses.down.sql" + +// TestMigration0072RecordsSurvivalInTheAddressTable is a pure-Go, +// DB-independent check of the frozen migration bytes: it runs even where no +// `dolt` binary is available. +// +// An address's survival is RECORDED on its own row, not inferred from other +// rows (gastownhall/beads#6664, bee-ghosttrack review 5360880888, Major 1): +// carried_from names the root address of the lineage a token-scheme change +// carried the row from, and gone_at_epoch is the epoch the store lost it in. +// Both are NULL for a fresh mint. The index serves the two lookups that +// follow a lineage (the newest carry of a root, and every carry of a root to +// lose), and the CHECK is the database-level tripwire against a stale or +// rewound epoch being written into a loss marker. It is >=, not >, because a +// carry row is born at the new epoch and can be lost in that same epoch. +// +// The migration is edited IN PLACE and gains no ALTER and no second slot: +// 0072 has never been applied to a store, and an ALTER over an already-applied +// copy of the earlier shape would be a migration slot nothing may claim. +func TestMigration0072RecordsSurvivalInTheAddressTable(t *testing.T) { + upSQL, err := MigrationSQL(migration0072Up) + if err != nil { + t.Fatalf("MigrationSQL(%s) error = %v, want the migration file to exist", migration0072Up, err) + } + for _, want := range []string{ + "CREATE TABLE IF NOT EXISTS epoch_minted_addresses", + "address VARCHAR(255) NOT NULL,", + "store_id VARCHAR(255) NOT NULL,", + "minted_id VARCHAR(255) NOT NULL,", + "minted_epoch INT NOT NULL,", + "minted_at DATETIME NOT NULL,", + "carried_from VARCHAR(255) NULL,", + "gone_at_epoch INT NULL,", + "PRIMARY KEY (address),", + "INDEX idx_epoch_minted_addresses_carried_from (carried_from),", + "CONSTRAINT ck_epoch_minted_addresses_gone_not_before_mint CHECK (gone_at_epoch IS NULL OR gone_at_epoch >= minted_epoch)", + } { + if !strings.Contains(upSQL, want) { + t.Errorf("0072 up migration missing %q\nfull SQL:\n%s", want, upSQL) + } + } + + // Only the statements count here: the header comments describe the + // migration in words that would otherwise trip these checks. + statements := strings.ToUpper(stripSQLLineComments(upSQL)) + if strings.Contains(statements, "ALTER TABLE") { + t.Error("0072 up migration must not ALTER anything: the survival columns belong to its own CREATE TABLE, because the migration is edited in place while no store has applied it") + } + if strings.Contains(statements, "PREPARE") { + t.Error("0072 up migration must stay plain DDL: CREATE TABLE IF NOT EXISTS needs no guarded PREPARE, and adding one would put it on the pre-2.3 CLI hazard path") + } + if got := strings.Count(statements, "CREATE TABLE"); got != 1 { + t.Errorf("0072 up migration has %d CREATE TABLE statements, want exactly 1", got) + } + + // Only migrations/*.up.sql is embedded, so the down side reads from disk by + // package-relative path, as migration 0067's test does. + downBytes, err := os.ReadFile("migrations/" + migration0072Down) + if err != nil { + t.Fatalf("read %s: %v, want the migration file to exist", migration0072Down, err) + } + if !strings.Contains(string(downBytes), "DROP TABLE IF EXISTS epoch_minted_addresses") { + t.Errorf("0072 down migration must drop epoch_minted_addresses, got:\n%s", downBytes) + } +} + +// TestMigration0072RecordsSurvivalInTheAddressTableThroughDoltCLI applies the +// full migration bundle through a real `dolt` binary (skipped without one, see +// testutil.RequireDoltBinary) and checks what a pure-Go SQL-text check cannot: +// the column types and nullability Dolt reports, the index it actually built, +// and that the CHECK is enforced on both INSERT and UPDATE, with the boundary +// it is meant to have. +func TestMigration0072RecordsSurvivalInTheAddressTableThroughDoltCLI(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := t.TempDir() + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + runDoltSQL(t, dir, AllMigrationsSQL()) + + requireDoltColumnShape(t, dir, "epoch_minted_addresses", "carried_from", "varchar(255)", "YES") + requireDoltDataType(t, dir, "epoch_minted_addresses", "gone_at_epoch", "int", "YES") + // The five columns the mint writes keep their shape, so the mint's + // five-column INSERT is unchanged. + requireDoltColumnShape(t, dir, "epoch_minted_addresses", "address", "varchar(255)", "NO") + requireDoltColumnShape(t, dir, "epoch_minted_addresses", "minted_epoch", "int", "NO") + requireDoltNoRows(t, dir, "SELECT address FROM epoch_minted_addresses", "epoch_minted_addresses") + requireDoltCount(t, dir, + `SELECT COUNT(*) AS c FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'epoch_minted_addresses' AND INDEX_NAME = 'idx_epoch_minted_addresses_carried_from' AND COLUMN_NAME = 'carried_from'`, + "1") + + // A fresh mint writes the five original columns and is a root row: both + // survival columns come out NULL with no change to the INSERT. + runDoltSQL(t, dir, `INSERT INTO epoch_minted_addresses (address, store_id, minted_id, minted_epoch, minted_at) VALUES ('root-a', 'store', 'record-a', 3, '2026-09-01 00:00:00')`) + requireDoltCount(t, dir, `SELECT COUNT(*) AS c FROM epoch_minted_addresses WHERE address = 'root-a' AND carried_from IS NULL AND gone_at_epoch IS NULL`, "1") + + // A carry row names the lineage root. + runDoltSQL(t, dir, `INSERT INTO epoch_minted_addresses (address, store_id, minted_id, minted_epoch, minted_at, carried_from) VALUES ('carry-a', 'store', 'record-a', 5, '2026-09-01 00:00:00', 'root-a')`) + requireDoltCount(t, dir, `SELECT COUNT(*) AS c FROM epoch_minted_addresses WHERE carried_from = 'root-a'`, "1") + + // The CHECK: an epoch lost in may not precede the epoch minted in. + if err := runDoltSQLExpectingError(t, dir, `UPDATE epoch_minted_addresses SET gone_at_epoch = 2 WHERE address = 'root-a'`); err == nil { + t.Error("UPDATE gone_at_epoch below minted_epoch succeeded, want the CHECK constraint to reject it") + } + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO epoch_minted_addresses (address, store_id, minted_id, minted_epoch, minted_at, gone_at_epoch) VALUES ('bad-b', 'store', 'record-b', 4, '2026-09-01 00:00:00', 3)`); err == nil { + t.Error("INSERT with gone_at_epoch below minted_epoch succeeded, want the CHECK constraint to reject it") + } + // Equal is allowed: a carry row is born at the new epoch and can be lost in + // that same epoch. + runDoltSQL(t, dir, `UPDATE epoch_minted_addresses SET gone_at_epoch = 3 WHERE address = 'root-a'`) + requireDoltCount(t, dir, `SELECT COUNT(*) AS c FROM epoch_minted_addresses WHERE address = 'root-a' AND gone_at_epoch = 3`, "1") + // A later epoch is allowed too. + runDoltSQL(t, dir, `UPDATE epoch_minted_addresses SET gone_at_epoch = 9 WHERE address = 'carry-a'`) + requireDoltCount(t, dir, `SELECT COUNT(*) AS c FROM epoch_minted_addresses WHERE address = 'carry-a' AND gone_at_epoch = 9`, "1") + + // The address stays the primary key. + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO epoch_minted_addresses (address, store_id, minted_id, minted_epoch, minted_at) VALUES ('root-a', 'store', 'record-a', 3, '2026-09-01 00:00:00')`); err == nil { + t.Error("duplicate address insert succeeded, want primary key violation") + } +} + +// stripSQLLineComments drops "--" comment lines so a check over a migration's +// statements is not tripped by the prose in its header. +func stripSQLLineComments(sqlText string) string { + var kept []string + for _, line := range strings.Split(sqlText, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + kept = append(kept, line) + } + return strings.Join(kept, "\n") +} diff --git a/internal/storage/schema/migrations/0072_add_epoch_minted_addresses.down.sql b/internal/storage/schema/migrations/0072_add_epoch_minted_addresses.down.sql new file mode 100644 index 000000000..af8d83eea --- /dev/null +++ b/internal/storage/schema/migrations/0072_add_epoch_minted_addresses.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS epoch_minted_addresses; diff --git a/internal/storage/schema/migrations/0072_add_epoch_minted_addresses.up.sql b/internal/storage/schema/migrations/0072_add_epoch_minted_addresses.up.sql new file mode 100644 index 000000000..ec946cf8e --- /dev/null +++ b/internal/storage/schema/migrations/0072_add_epoch_minted_addresses.up.sql @@ -0,0 +1,64 @@ +-- epoch_minted_addresses: durable storage for R20 epoch-transition +-- enforcement (gastownhall/beads#5898 revision 9, +-- #6136). Independent of store_epoch (0067, #6135) -- that table holds only +-- the store-wide singleton epoch counter (the current generation number); +-- this table tracks the individual addresses minted under each generation, +-- which store_epoch has no room to hold. It is also independent of +-- expected_revision_records (R16, #6133) and any RetentionFixture/R17 +-- table: R20 epoch reasoning is evaluated on its own, without consulting +-- retention/erasure state, and this migration adds no R17 resolve, remove, +-- hold, force-remove, erase, or mint logic. +-- +-- One row per minted address, not one row per id: MintUnderEpoch is +-- deterministic in (store_id, id, epoch) -- see epochAddress() in +-- internal/storage/issueops/epoch_cas.go -- so minting the same id again +-- under the same epoch reproduces the same address and finds the same +-- row, while a later mint of that id under a bumped epoch produces a +-- DIFFERENT address and adds a new row alongside the old one. Old rows are +-- kept: StillServes and Resolve answer for any address from its own row, +-- whatever epoch it was minted in. +-- +-- Survival is RECORDED on the row, never inferred from other rows +-- (gastownhall/beads#6664, bee-ghosttrack review 5360880888, Major 1). +-- gone_at_epoch is NULL while the store serves the address and otherwise the +-- epoch the store lost it in; it is set once and never cleared, so a lost +-- address stays lost. carried_from is NULL for a fresh mint, and for a row a +-- token-scheme change carried to a new epoch it is the address of the +-- lineage's ROOT (never the previous carry), so every member of a lineage is +-- one indexed lookup from the newest. The index on carried_from serves exactly +-- the two lookups that follow a lineage: the newest carry of a root, and every +-- carry of a root when the lineage is lost. The CHECK is a database-level +-- tripwire against a stale or rewound epoch being written into a loss marker; +-- it is >=, not >, because a carry row is born at the new epoch and can be lost +-- in that same epoch. +-- +-- address is the PRIMARY KEY: it is the deterministic token the fixture's +-- StillServes/Resolve/CurrentAddressFor hooks look addresses up by, so it +-- must be unique and indexed; it is never recomputed from the other columns +-- at read time; it is written once, whatever the epoch was at mint time. +-- +-- store_id and minted_id are VARCHAR(255) for parity with this schema's +-- other identifier columns. minted_epoch is INT to match store_epoch.epoch's +-- own type. minted_at is DATETIME, not a nanosecond-precision integer: unlike +-- expected_revision_records.change_at_nanos (R16, 0069 on that slice), no +-- R20 contract case compares minted_at for exact round-tripping -- it exists +-- for observability only, so it follows store_epoch.bumped_at's own +-- DATETIME precedent rather than needing BIGINT. +-- +-- Plain, unguarded CREATE TABLE IF NOT EXISTS: a brand-new main-plane table +-- with no ALTER, no PREPARE, and no dolt-ignored (clone-local) table +-- involved, so none of cli_migrations.go's CLI-bundle overrides, an +-- ignored/ twin, or a nondeterminism-allowlist entry apply +-- (scripts/check-migration-hygiene.sh checks B-E). +CREATE TABLE IF NOT EXISTS epoch_minted_addresses ( + address VARCHAR(255) NOT NULL, + store_id VARCHAR(255) NOT NULL, + minted_id VARCHAR(255) NOT NULL, + minted_epoch INT NOT NULL, + minted_at DATETIME NOT NULL, + carried_from VARCHAR(255) NULL, + gone_at_epoch INT NULL, + PRIMARY KEY (address), + INDEX idx_epoch_minted_addresses_carried_from (carried_from), + CONSTRAINT ck_epoch_minted_addresses_gone_not_before_mint CHECK (gone_at_epoch IS NULL OR gone_at_epoch >= minted_epoch) +); diff --git a/internal/storage/uow/expected_revision_contract_test.go b/internal/storage/uow/expected_revision_contract_test.go index 53f17947d..a1ccfa1b4 100644 --- a/internal/storage/uow/expected_revision_contract_test.go +++ b/internal/storage/uow/expected_revision_contract_test.go @@ -8,11 +8,21 @@ import ( ) // TestExpectedRevisionContract wires this leg into the R16/R17 -// expected-revision contract. Phase 0 leaves every hook nil in every -// backend's fixture kit (architecture §12), so each case below skips by -// name; this file exists so TestEveryLegWiresEveryRoleContract counts this -// leg, and so the cases start running for real the moment this leg's fixture -// kit grows a non-nil hook. +// expected-revision contract. Every hook is nil (architecture §12's Phase 0 +// default), so each case below skips by name; this file exists so +// TestEveryLegWiresEveryRoleContract counts this leg, and so the cases +// start running for real the moment this leg grows a real +// CompareAndSetVersion (gastownhall/beads#6358). +// +// This leg previously wired every hook to an honest "not implemented" stub +// instead of leaving them nil, so each case ran for real and +// failed for that documented reason rather than skipping. That premise — +// that a later slice would give this leg its CompareAndSetVersion — went +// stale once review moved the real implementation to +// gastownhall/beads#6358 instead; a stub whose failure message blames a +// bead that will never land it is worse than an honest skip, so this file +// reverts to nil hooks (gastownhall/beads#6664, bee-ghosttrack review +// 5268699223). func TestExpectedRevisionContract(t *testing.T) { ctx := context.Background() fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "erev"} diff --git a/internal/storage/uow/retention_epoch_contract_test.go b/internal/storage/uow/retention_epoch_contract_test.go index a2d8bcf00..91d3a5324 100644 --- a/internal/storage/uow/retention_epoch_contract_test.go +++ b/internal/storage/uow/retention_epoch_contract_test.go @@ -2,9 +2,11 @@ package uow import ( "context" + "fmt" "testing" "github.com/steveyegge/beads/backend/conformance" + storeops "github.com/steveyegge/beads/internal/storage/issueops" ) // TestRetentionContract wires this leg into the R20 retention contract. @@ -57,11 +59,28 @@ func TestRetentionContract(t *testing.T) { }) } -// TestEpochContract wires this leg into the R20 epoch contract. Same Phase 0 -// all-nil state as TestRetentionContract above. +// TestEpochContract runs the R20 epoch contract (gastownhall/beads#5898 +// revision 9, #6136) against the unit-of-work +// provider, which reaches the same internal/storage/issueops's epoch Tx +// functions the two store backends wrap — directly against the unit of +// work's own runner (the *baseUOW/base.tx.Runner() pattern +// expected_revision_contract_test.go's expectedRevisionUOWMutateOutside and +// commenter_contract_test.go's SeedCommentAt both use), NOT through the +// domain issue repository: EpochFixture's seven hooks are store-singleton/ +// administrative operations with no natural correspondence to +// IssueUseCase's per-issue CRUD/notification semantics, so unlike +// CompareAndSetVersion there is no domain/use-case method to route through +// in the first place — every hook below needs the escape hatch, not just +// one. +// +// One provider for the whole suite (each newUOWRoleFixtureProvider boots a +// real Dolt sql-server) and NO t.Parallel: this backend has no per-test +// copy-on-write branch, so store_epoch/epoch_minted_addresses are +// database-global and a parallel subtest would corrupt another subtest's +// row — same reasoning as TestExpectedRevisionContract's. func TestEpochContract(t *testing.T) { ctx := context.Background() - fixture := conformance.EpochFixture{IssuePrefix: "epch"} + fixture := newUOWEpochFixture(t, ctx, "epch") t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { conformance.RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) @@ -69,4 +88,184 @@ func TestEpochContract(t *testing.T) { t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { conformance.RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) }) + t.Run("TokenSchemeChangeCarriesLiveAddressesAndOnlyThose", func(t *testing.T) { + conformance.RunTokenSchemeChangeCarriesLiveAddressesAndOnlyThose(t, ctx, fixture) + }) + t.Run("ACarriedAddressKeepsResolvingAcrossLaterBumps", func(t *testing.T) { + conformance.RunACarriedAddressKeepsResolvingAcrossLaterBumps(t, ctx, fixture) + }) + t.Run("ALostAddressStaysGoneAcrossASchemeChange", func(t *testing.T) { + conformance.RunALostAddressStaysGoneAcrossASchemeChange(t, ctx, fixture) + }) +} + +// newUOWEpochFixture wires this leg's UnitOfWorkProvider into the R20 epoch +// contract. Like newUOWExpectedRevisionFixture, this fixture needs none of +// roleFixtureKit's CreateIssue/CreateWisp/QueryScalar/CountHistory hooks — +// it has no fields for them — so it does not route through +// newUOWRoleFixtureKit at all. +func newUOWEpochFixture(t *testing.T, ctx context.Context, prefix string) conformance.EpochFixture { + t.Helper() + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + return conformance.EpochFixture{ + IssuePrefix: prefix, + CurrentEpoch: epochUOWCurrentEpoch(provider), + BumpEpoch: epochUOWBumpEpoch(provider), + MintUnderEpoch: epochUOWMintUnderEpoch(provider), + StillServes: epochUOWStillServes(provider), + Resolve: epochUOWResolve(provider), + CurrentAddressFor: epochUOWCurrentAddressFor(provider), + LoseVersion: epochUOWLoseVersion(provider), + } +} + +// epochUOWRunner recovers the *baseUOW a given attempt's UnitOfWork actually +// is, the same type assertion expectedRevisionUOWMutateOutside makes: Tx. +// Runner() is unexported outside this package and none of the seven epoch +// operations has a domain/use-case method to reach it through instead. +func epochUOWRunner(uw UnitOfWork) (storeops.DBTX, error) { + base, ok := uw.(*baseUOW) + if !ok { + return nil, fmt.Errorf("epoch CAS: unit of work %T does not expose the runner the epoch Tx functions need", uw) + } + return base.tx.Runner(), nil +} + +func epochUOWCurrentEpoch(provider UnitOfWorkProvider) func(ctx context.Context, storeID string) (int, error) { + return func(ctx context.Context, storeID string) (int, error) { + return RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (int, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return 0, err + } + return storeops.CurrentEpochInTx(ctx, runner, storeID) + }) + } +} + +// epochUOWBumpEpoch chooses the bump by the typed trigger: only a +// token-scheme-change bump carries live addresses to the new epoch, and the +// other two triggers move the counter alone. The choice lives here, in the +// contract-test translation layer, because the shared body has no switch on +// the reason string. +func epochUOWBumpEpoch(provider UnitOfWorkProvider) func(ctx context.Context, storeID string, trigger conformance.EpochBumpTrigger) (int, error) { + return func(ctx context.Context, storeID string, trigger conformance.EpochBumpTrigger) (int, error) { + return RunTxResult(ctx, provider, func(ctx context.Context, uw UnitOfWork) (int, string, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return 0, "", err + } + bump := storeops.BumpEpochInTx + if trigger == conformance.EpochBumpTriggerTokenSchemeChange { + bump = storeops.BumpEpochCarryingInTx + } + epoch, err := bump(ctx, runner, storeID, trigger.String()) + if err != nil { + return 0, "", err + } + return epoch, fmt.Sprintf("bd: bump epoch for %s (%s)", storeID, trigger.String()), nil + }) + } +} + +func epochUOWMintUnderEpoch(provider UnitOfWorkProvider) func(ctx context.Context, storeID, id string) (conformance.Address, error) { + return func(ctx context.Context, storeID, id string) (conformance.Address, error) { + address, err := RunTxResult(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, string, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return "", "", err + } + address, err := storeops.MintUnderEpochInTx(ctx, runner, storeID, id) + if err != nil { + return "", "", err + } + return address, fmt.Sprintf("bd: mint %s under epoch for %s", id, storeID), nil + }) + if err != nil { + return "", err + } + return conformance.Address(address), nil + } +} + +func epochUOWStillServes(provider UnitOfWorkProvider) func(ctx context.Context, storeID string, address conformance.Address) (bool, error) { + return func(ctx context.Context, storeID string, address conformance.Address) (bool, error) { + return RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (bool, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return false, err + } + return storeops.StillServesInTx(ctx, runner, storeID, string(address)) + }) + } +} + +func epochUOWResolve(provider UnitOfWorkProvider) func(ctx context.Context, storeID string, address conformance.Address) (conformance.RetentionAnswer, error) { + return func(ctx context.Context, storeID string, address conformance.Address) (conformance.RetentionAnswer, error) { + result, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (storeops.EpochResolveResult, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return storeops.EpochResolveResult{}, err + } + return storeops.ResolveEpochInTx(ctx, runner, storeID, string(address)) + }) + if err != nil { + return conformance.RetentionAnswer{}, err + } + return epochResolveResultToConformance(result), nil + } +} + +func epochUOWCurrentAddressFor(provider UnitOfWorkProvider) func(ctx context.Context, storeID string, oldAddress conformance.Address) (conformance.Address, error) { + return func(ctx context.Context, storeID string, oldAddress conformance.Address) (conformance.Address, error) { + address, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return "", err + } + return storeops.CurrentAddressForInTx(ctx, runner, storeID, string(oldAddress)) + }) + if err != nil { + return "", err + } + return conformance.Address(address), nil + } +} + +func epochUOWLoseVersion(provider UnitOfWorkProvider) func(ctx context.Context, storeID string, address conformance.Address) error { + return func(ctx context.Context, storeID string, address conformance.Address) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + runner, err := epochUOWRunner(uw) + if err != nil { + return "", err + } + if err := storeops.LoseVersionInTx(ctx, runner, storeID, string(address)); err != nil { + return "", err + } + return fmt.Sprintf("bd: lose version %s for %s", address, storeID), nil + }) + } +} + +// epochRestrictionToConformance translates this leg's local +// storeops.EpochRestriction into backend/conformance's own Restriction +// vocabulary — the contract-test file is the translation boundary, per +// epoch_cas.go's package doc. +func epochRestrictionToConformance(r storeops.EpochRestriction) conformance.Restriction { + switch r { + case storeops.EpochRestrictionLive: + return conformance.RestrictionLive + case storeops.EpochRestrictionGoneReorganization: + return conformance.RestrictionGoneReorganization + default: + return conformance.RestrictionUnknown + } +} + +func epochResolveResultToConformance(r storeops.EpochResolveResult) conformance.RetentionAnswer { + return conformance.RetentionAnswer{ + Restriction: epochRestrictionToConformance(r.Restriction), + ProducingStore: r.ProducingStore, + Epoch: r.Epoch, + } } From 3488dc4536ee4438c424a8d65031f379744ce67f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 03:11:14 +0000 Subject: [PATCH 16/54] fix(init): `bd init --server --external` starts no server; a failed `bd init --server` can be retried (#7674) * fix(init): admit a retried `bd init --server` over an empty .beads/dolt A first `bd init --server` that cannot reach its server fails after creating the local Dolt root, leaving an empty .beads/dolt, no .local_version witness, and no config.yaml/metadata.json recording the server choice. The legacy-upgrade guard runs before init writes that choice, so it saw an embedded-looking workspace with a local Dolt root and refused every retry as a "legacy Dolt workspace"; no bd command could repair it. guardLegacyUpgradeWorkspaceForInit(beadsDir, initSelectsServer) carries init's already-resolved server intent into the guard. It is consulted for exactly one decision: an empty, witness-less .beads/dolt is admitted, the same admission a workspace that records server mode already gets (#6935). It is deliberately not folded into the workspace's own server mode, so a non-empty root with an unreadable witness and an embedded repository beside a non-empty root keep their current outcomes. guardLegacyUpgradeWorkspace is now the initSelectsServer=false wrapper, and only init's main guard call passes the flag; the proxied-server init path keeps the plain wrapper. Ported from the versioned-beads fork (donnabox/codex/preview3-source- catchup-46aa), commits bc8d72182a (red) and 5bdfbfe5e2 (green). The fork's graph_mode refusal in the same function is graph-preview-only and is not carried here. Tests: TestLegacyUpgradeGuardForInitAdmitsEmptyDoltRoot and TestLegacyUpgradeGuardForInitAdmissionStaysNarrow pin the admission and its narrowness at the guard; TestInitServerFlagAdmitsEmptyDoltRootWithoutConfig and TestInitServerFlagRetryAfterFailedFirstAttempt run the real binary against the test Dolt server and fail on main with the legacy refusal. Co-authored-by: Jim Wordelman Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * fix(init): `bd init --server --external` must not auto-start a server `--external` declares the Dolt server managed outside bd ("skip server startup", per its help), but init builds its dolt.Config by hand with `AutoStart: initServerMode && BEADS_DOLT_AUTO_START != "0"`, so the flag never reached the auto-start decision. A `bd init --server --external` that named no port therefore ran dolt and tried to start a repo-local sql-server. A named port was already declared external by rule 2c (#5934), but its failure still advised `bd dolt start`, the one thing --external says bd must not do, and each route (host:port, socket, shared server, auto-start pinned off) worded the failure differently. - AutoStart is now `initServerMode && !externalServer && ...`. - externalServerUnreachableMessage rewrites only a failed dial (*net.OpError with Op "dial") under --external: it names the endpoint that was dialed and says bd did not start a server. A dial to port 0 means no port was named, so that message asks for --server-port or BEADS_DOLT_SERVER_PORT and offers no default (GH#2098, GH#2372). Any other open failure keeps its own text. Ported from the versioned-beads fork (donnabox/codex/preview3-source- catchup-46aa), commits 234ca3d502 and a601a80e77 (red/green), 03a64a1aba (socket helper rename), and a6ca0536c9 and 397fc4c69e (red/green: no-port arm and wording). The fork's graph-preview dispatch at the top of init and its --skip-hooks help text change are graph-only and not carried. The TestInitExternalProvisionerShape doc comment is corrected: on main every provisioner cell is a control (a named port is already external), not a reproduction. Tests: TestInitServerExternalDeadPortStartsNoServer, TestInitServerExternalNoPortStartsNoServer and TestInitExternalUnreachableMessage run the real binary against a dolt shim that records every launch and fail on main; TestInitSharedServerExternalDeadPortStillFails, TestInitServerWithoutNamedPortStillStartsServer and TestInitExternalProvisionerShape are controls; and TestExternalServerUnreachableMessage pins which errors are rewritten. Co-authored-by: Jim Wordelman Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Jim Wordelman Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 18 + cmd/bd/BUILD.bazel | 4 + cmd/bd/init.go | 49 +- cmd/bd/init_external_autostart_test.go | 430 ++++++++++++++++++ cmd/bd/init_external_message_test.go | 97 ++++ cmd/bd/legacy_upgrade_guard.go | 22 + ...legacy_upgrade_guard_init_flag_e2e_test.go | 87 ++++ cmd/bd/legacy_upgrade_guard_init_flag_test.go | 143 ++++++ 8 files changed, 847 insertions(+), 3 deletions(-) create mode 100644 cmd/bd/init_external_autostart_test.go create mode 100644 cmd/bd/init_external_message_test.go create mode 100644 cmd/bd/legacy_upgrade_guard_init_flag_e2e_test.go create mode 100644 cmd/bd/legacy_upgrade_guard_init_flag_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d4d31e9..4dbfca8b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -763,6 +763,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd init --server --external` no longer starts a Dolt server of its own.** + Init builds its store config by hand and turned auto-start on for every + server init, so an `--external` init that named no port launched a + repo-local `dolt sql-server`, although `--external` says the server is + managed outside bd. `--external` now turns auto-start off. When the + external server cannot be reached, init gives one message on every route + (host and port, socket, shared server, auto-start pinned off): it names the + endpoint it dialed, says that bd did not start a server, and no longer + advises `bd dolt start`. With no port named at all, it asks for one + (`--server-port` or `BEADS_DOLT_SERVER_PORT`) instead of offering a default. +- **A failed `bd init --server` can be retried.** A first attempt that could + not reach its server left an empty `.beads/dolt` and nothing on disk that + records the server choice, so the legacy-upgrade guard refused every retry + as a "legacy Dolt workspace" and no bd command could repair it. When init + itself selects server mode, the guard now admits exactly that root: empty, + with no `.local_version` witness. A pre-1.0 witness, or a `.beads/dolt` that + holds anything, is still refused, and commands other than `bd init` are + decided as before. - **Time-bound filters and the keyset cursor compare the same instant on every engine, whatever zone they are given in.** The storage filter builder binds every time bound (`created_*`, `updated_*`, `closed_*`, `started_*`, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6eb4de6e7..4a4e51b9d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -891,6 +891,8 @@ go_test( "init_commit_test.go", "init_embedded_test.go", "init_explicit_beads_dir_embedded_test.go", + "init_external_autostart_test.go", + "init_external_message_test.go", "init_gateway_test.go", "init_git_bootstrap_routing_test.go", "init_guard_force_e2e_test.go", @@ -939,6 +941,8 @@ go_test( "legacy_upgrade_guard_e2e_test.go", "legacy_upgrade_guard_empty_root_test.go", "legacy_upgrade_guard_external_server_test.go", + "legacy_upgrade_guard_init_flag_e2e_test.go", + "legacy_upgrade_guard_init_flag_test.go", "legacy_upgrade_guard_test.go", "legacy_upgrade_guard_witness_test.go", "linear_roundtrip_test.go", diff --git a/cmd/bd/init.go b/cmd/bd/init.go index a5b9cd9c8..bca3e336e 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "net" "os" "os/exec" "os/user" @@ -226,6 +227,40 @@ func shouldInitSharedGlobalDB(sharedServer, sharedServerMode, gateway bool) bool return (sharedServer || sharedServerMode) && !gateway } +// externalServerUnreachableMessage returns the message for an --external init +// whose server could not be dialed, or "" when err is any other failure. +// +// --external declares the server managed outside bd, so init starts none. The +// store's own advice for a dead local server, "bd dolt start", would start the +// very server --external says bd must not, and the store words this failure +// differently per route (shared server, socket, remote host, auto-start pinned +// off). One message serves them all: it names the endpoint that was dialed and +// says that bd did not start a server. +// +// A dial to port 0 is the exception. The store leaves a zero port zero for +// auto-start to fill in, so it means nothing named a port: the address is a +// placeholder, not an endpoint. The message says so and how to name one, and +// offers no default, since on a shared machine whatever listens on a default +// port may belong to another project (GH#2098, GH#2372). +func externalServerUnreachableMessage(err error) string { + var opErr *net.OpError + if !errors.As(err, &opErr) || opErr.Op != "dial" { + return "" + } + if addr, ok := opErr.Addr.(*net.TCPAddr); ok && addr.Port == 0 { + return fmt.Sprintf("Error: failed to open Dolt store: %v\n\n"+ + "--external declares the Dolt server externally managed, so bd did not start one, and no port was named for it.\n"+ + "Name the server's port with --server-port or BEADS_DOLT_SERVER_PORT.\n", opErr) + } + endpoint := "the configured endpoint" + if opErr.Addr != nil { + endpoint = opErr.Addr.String() + } + return fmt.Sprintf("Error: failed to open Dolt store: %v\n\n"+ + "--external declares the Dolt server at %s externally managed, so bd did not start one.\n"+ + "Start that server, or point init at the right endpoint.\n", opErr, endpoint) +} + // warnHalfIdentifiedSubstrate reports a substrate carrying one identity marker // and not the other. // @@ -970,7 +1005,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // before init's existing-workspace checks so even --force cannot create // or rewrite state beside a source that has not been preserved. if beadsDir := resolveInitBeadsDir(); beadsDir != "" { - if err := guardLegacyUpgradeWorkspace(beadsDir); err != nil { + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, initServerMode); err != nil { return err } } @@ -1554,7 +1589,9 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } // Build config. Beads always uses dolt sql-server. - // AutoStart is always enabled during init — we need a server to initialize the database. + // AutoStart is enabled during init — we need a server to initialize the database — + // except under --external: that server is managed outside bd, so bd must not start + // one of its own on the endpoint the caller named. // // Port resolution for init: use ONLY project-local sources (env var, port file) // to prevent cross-project data leakage (GH#2336). DefaultConfig falls through @@ -1602,7 +1639,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): ProxiedServer: initProxiedServer, CreateIfMissing: true, // bd init is the only path that should create databases OpenedByInit: true, // shapes the identity-mismatch advice (GH#5558) - AutoStart: initServerMode && os.Getenv("BEADS_DOLT_AUTO_START") != "0", + AutoStart: initServerMode && !externalServer && os.Getenv("BEADS_DOLT_AUTO_START") != "0", ServerTLS: initDoltServerTLSFromEnv(), } if serverHost != "" { @@ -1727,6 +1764,12 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } return &exitError{Code: 1} } + if externalServer { + if msg := externalServerUnreachableMessage(err); msg != "" { + fmt.Fprint(os.Stderr, msg) + return &exitError{Code: 1} + } + } fmt.Fprintf(os.Stderr, "Error: failed to open Dolt store: %v\n", err) return &exitError{Code: 1} } diff --git a/cmd/bd/init_external_autostart_test.go b/cmd/bd/init_external_autostart_test.go new file mode 100644 index 000000000..b08e6c3be --- /dev/null +++ b/cmd/bd/init_external_autostart_test.go @@ -0,0 +1,430 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/configfile" +) + +// `bd init --external` declares that the Dolt server is managed outside bd, so +// init must never start one. These tests own one boundary risk that no lower +// seam can show: init builds its dolt config by hand, so the flag reaches the +// auto-start decision only if init carries it there. They run the real bd +// binary against a dolt shim that records every launch, which makes the +// outcome observable without a real server and leaves nothing running. + +// externalInitFixture is one fresh project plus a dolt shim for `bd init` +// attempts. The shim sits first on PATH, records every invocation and fails. +type externalInitFixture struct { + t *testing.T + repoDir string + home string + shimDir string + launches string +} + +type externalInitResult struct { + output string // combined stdout and stderr + failed bool // bd exited non-zero + launches string // dolt invocations the shim saw, one per line +} + +func newExternalInitFixture(t *testing.T) *externalInitFixture { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("the dolt shim is a POSIX shell script") + } + f := &externalInitFixture{t: t, repoDir: t.TempDir(), home: t.TempDir(), shimDir: t.TempDir()} + f.launches = filepath.Join(f.shimDir, "invocations") + shim := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$EXTERNAL_INIT_SHIM_LOG\"\nexit 1\n" + if err := os.WriteFile(filepath.Join(f.shimDir, "dolt"), []byte(shim), 0o755); err != nil { + t.Fatal(err) + } + initGitRepo(t, f.repoDir) + return f +} + +// run executes bd in the project. autoStart is the value of +// BEADS_DOLT_AUTO_START, or "" to leave it unset: an unpinned environment is +// what most callers run in, so it is the one that matters here. +func (f *externalInitFixture) run(autoStart string, args ...string) externalInitResult { + f.t.Helper() + env := append(envWithoutBeadsStorageSettings(), + "HOME="+f.home, + "XDG_CONFIG_HOME="+filepath.Join(f.home, ".config"), + "BEADS_TEST_IGNORE_REPO_CONFIG=1", + "BD_DISABLE_METRICS=1", + "PATH="+f.shimDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "EXTERNAL_INIT_SHIM_LOG="+f.launches, + ) + if autoStart != "" { + env = append(env, "BEADS_DOLT_AUTO_START="+autoStart) + } + out, err := runExternalServerBD(f.t, f.repoDir, env, args...) + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + f.t.Fatalf("bd %s did not run: %v\n%s", strings.Join(args, " "), err, out) + } + launches, _ := os.ReadFile(f.launches) + return externalInitResult{output: out, failed: err != nil, launches: strings.TrimSpace(string(launches))} +} + +// externalEndpoint is the server an --external init is pointed at. +type externalEndpoint struct { + flags []string // --server-host/--server-port, or --server-socket + network string // "tcp" or "unix" + addr string // host:port or the socket path, as bd reports it +} + +func tcpEndpoint(port int) externalEndpoint { + return externalEndpoint{ + flags: []string{"--server-host", "127.0.0.1", "--server-port", strconv.Itoa(port)}, + network: "tcp", + addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), + } +} + +func unixEndpoint(path string) externalEndpoint { + return externalEndpoint{flags: []string{"--server-socket", path}, network: "unix", addr: path} +} + +// externalInitSocketPath returns an unused unix socket path. Socket paths are +// capped near 104 bytes and t.TempDir() nests too deeply under some runners. +func externalInitSocketPath(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "bdx") + if err != nil { + t.Fatal(err) + } + if len(filepath.Join(dir, "dolt.sock")) > 100 { + _ = os.RemoveAll(dir) + if dir, err = os.MkdirTemp("/tmp", "bdx"); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "dolt.sock") +} + +// serveUnixProxy listens on a fresh unix socket and forwards each connection +// to target, so bd sees a reachable server at a socket path. The returned +// func reports how many connections bd made through it. +func serveUnixProxy(t *testing.T, target string) (string, func() int) { + t.Helper() + path := externalInitSocketPath(t) + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatalf("listen on %s: %v", path, err) + } + t.Cleanup(func() { _ = ln.Close() }) + var mu sync.Mutex + accepted := 0 + go func() { + for { + client, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + accepted++ + mu.Unlock() + go func() { + defer client.Close() + upstream, err := net.Dial("tcp", target) + if err != nil { + return + } + defer upstream.Close() + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(upstream, client); done <- struct{}{} }() + go func() { _, _ = io.Copy(client, upstream); done <- struct{}{} }() + <-done // either side closing ends the session; the deferred closes release the other copy + }() + } + }() + return path, func() int { mu.Lock(); defer mu.Unlock(); return accepted } +} + +// requireNoLaunch fails when bd tried to run dolt at all: an externally +// managed server is never started by bd, and the shim is how a start shows. +func requireNoLaunch(t *testing.T, res externalInitResult) { + t.Helper() + if res.launches != "" { + t.Errorf("bd launched dolt although the server is externally managed (dolt %s):\n%s", + strings.ReplaceAll(res.launches, "\n", "; dolt "), res.output) + } +} + +// requireExternalUnreachableMessage checks the one wording an --external init +// gives for an endpoint it cannot reach: it names the endpoint, says the server +// is declared external and that bd did not start one, and never advises +// `bd dolt start`, which would start the server --external says bd must not. +func requireExternalUnreachableMessage(t *testing.T, output, endpoint string) { + t.Helper() + for _, want := range []string{endpoint, "externally managed", "did not start"} { + if !strings.Contains(output, want) { + t.Errorf("output does not contain %q:\n%s", want, output) + } + } + if strings.Contains(output, "bd dolt start") { + t.Errorf("output advises `bd dolt start`:\n%s", output) + } +} + +func requireNothingListening(t *testing.T, network, addr string) { + t.Helper() + conn, err := net.DialTimeout(network, addr, time.Second) + if err == nil { + _ = conn.Close() + t.Errorf("something accepts connections on %s %s after init", network, addr) + } +} + +// requireDoltRootEmpty fails when .beads/dolt holds anything: init leaves at +// most an empty marker directory for an external server, so any entry is a +// server root a local dolt created. +func requireDoltRootEmpty(t *testing.T, repoDir string) { + t.Helper() + entries, err := os.ReadDir(filepath.Join(repoDir, ".beads", "dolt")) + if errors.Is(err, os.ErrNotExist) { + return + } + if err != nil { + t.Fatal(err) + } + if len(entries) > 0 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf(".beads/dolt holds %v, which init did not put there", names) + } +} + +// TestInitServerExternalDeadPortStartsNoServer is the end-to-end boundary for +// --external: a dead endpoint must fail loudly instead of making bd launch a +// dolt of its own on the port the caller named. +func TestInitServerExternalDeadPortStartsNoServer(t *testing.T) { + f := newExternalInitFixture(t) + ep := tcpEndpoint(freeLoopbackPort(t)) + + res := f.run("", append(append([]string{"init", "--quiet", "--server", "--external"}, ep.flags...), + "--prefix", "x", "--skip-hooks", "--skip-agents")...) + + if !res.failed { + t.Errorf("bd init --server --external succeeded against a dead endpoint %s:\n%s", ep.addr, res.output) + } + requireNoLaunch(t, res) + requireExternalUnreachableMessage(t, res.output, ep.addr) + requireNothingListening(t, ep.network, ep.addr) + requireDoltRootEmpty(t, f.repoDir) +} + +// TestInitServerExternalNoPortStartsNoServer is the --external boundary that +// rule 2c (#5934) cannot see: the argv and the environment name no port, so +// nothing but the flag declares the server external. A server init in that +// shape launches a repo-local dolt by design (see +// TestInitServerWithoutNamedPortStillStartsServer), so with --external it must +// fail instead, and say that bd did not start one. There is no endpoint to +// name, so the wording is checked without one: the child inherits +// BEADS_TEST_MODE, under which the store dials port 1 rather than leave an +// unnamed port at 0, so the port-0 wording itself is pinned by +// TestExternalServerUnreachableMessage. +func TestInitServerExternalNoPortStartsNoServer(t *testing.T) { + f := newExternalInitFixture(t) + + res := f.run("", "init", "--quiet", "--server", "--external", "--prefix", "x", "--skip-hooks", "--skip-agents") + + if !res.failed { + t.Errorf("bd init --server --external succeeded with no server named:\n%s", res.output) + } + requireNoLaunch(t, res) + requireDoltRootEmpty(t, f.repoDir) + for _, want := range []string{"externally managed", "did not start"} { + if !strings.Contains(res.output, want) { + t.Errorf("output does not contain %q:\n%s", want, res.output) + } + } + if strings.Contains(res.output, "bd dolt start") { + t.Errorf("output advises `bd dolt start`:\n%s", res.output) + } +} + +// TestInitSharedServerExternalDeadPortStillFails pins the sibling route that +// already honors --external: the same dead endpoint under --shared-server +// exits non-zero without launching dolt. Without it the test above could pass +// because every init fails here, not because --server honors the flag. +func TestInitSharedServerExternalDeadPortStillFails(t *testing.T) { + f := newExternalInitFixture(t) + ep := tcpEndpoint(freeLoopbackPort(t)) + + res := f.run("", append(append([]string{"init", "--quiet", "--shared-server", "--external"}, ep.flags...), + "--prefix", "x", "--skip-hooks", "--skip-agents")...) + + if !res.failed { + t.Errorf("bd init --shared-server --external succeeded against a dead endpoint %s:\n%s", ep.addr, res.output) + } + requireNoLaunch(t, res) +} + +// TestInitServerWithoutNamedPortStillStartsServer pins the other half of the +// contract: a --server init that names no port is where bd starts a repo-local +// dolt by design, so the no-launch assertions above cannot pass merely because +// nothing can ever launch one. It names no port, in argv or in the environment +// (envWithoutBeadsStorageSettings drops every BEADS_DOLT_* variable): a named +// port is declared external by rule 2c (#5934) with or without --external, so +// it would launch nothing here either way. +func TestInitServerWithoutNamedPortStillStartsServer(t *testing.T) { + f := newExternalInitFixture(t) + + res := f.run("", "init", "--quiet", "--server", "--prefix", "x", "--skip-hooks", "--skip-agents") + + if res.launches == "" { + t.Errorf("bd init --server naming no port never reached dolt, so auto-start regressed:\n%s", res.output) + } +} + +// TestInitExternalUnreachableMessage covers the wording across the routes an +// --external init can take. Every one of them gives the same message, whether +// auto-start is pinned off by the environment, the server is a shared one, or +// the endpoint is a socket. +func TestInitExternalUnreachableMessage(t *testing.T) { + cases := []struct { + name string + mode string + autoStart string + endpoint func(t *testing.T) externalEndpoint + }{ + {"server host:port with auto-start pinned off", "--server", "0", + func(t *testing.T) externalEndpoint { return tcpEndpoint(freeLoopbackPort(t)) }}, + {"shared-server host:port", "--shared-server", "", + func(t *testing.T) externalEndpoint { return tcpEndpoint(freeLoopbackPort(t)) }}, + {"server socket", "--server", "", + func(t *testing.T) externalEndpoint { return unixEndpoint(externalInitSocketPath(t)) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newExternalInitFixture(t) + ep := tc.endpoint(t) + + res := f.run(tc.autoStart, append(append([]string{"init", "--quiet", tc.mode, "--external"}, ep.flags...), + "--prefix", "x", "--skip-hooks", "--skip-agents")...) + + if !res.failed { + t.Fatalf("bd init succeeded against a dead endpoint %s:\n%s", ep.addr, res.output) + } + requireExternalUnreachableMessage(t, res.output, ep.addr) + }) + } +} + +// TestInitExternalProvisionerShape runs init with the argv a provisioner of an +// externally managed server uses, in both endpoint forms, with auto-start +// unset and pinned off, against a server that is up and one that is not. Every +// cell is a control: a port named with --server-port is already declared +// external by rule 2c (#5934), and a socket is never auto-started, so none of +// them launched dolt before init honored --external either. They pin that the +// provisioner shape keeps launching nothing, keeps failing against a dead +// endpoint, and still records the endpoint it was given. The reachable cells +// need the Dolt test server and skip without it. +func TestInitExternalProvisionerShape(t *testing.T) { + for _, form := range []string{"host-port", "socket"} { + for _, autoStart := range []string{"unset", "0"} { + for _, reachable := range []bool{false, true} { + state := "unreachable" + if reachable { + state = "reachable" + } + t.Run(fmt.Sprintf("%s/auto-start=%s/%s", form, autoStart, state), func(t *testing.T) { + runProvisionerCell(t, form, autoStart, reachable) + }) + } + } + } +} + +func runProvisionerCell(t *testing.T, form, autoStart string, reachable bool) { + t.Helper() + if reachable { + skipIfNoDolt(t) + } + f := newExternalInitFixture(t) + + var ep externalEndpoint + var socketConnections func() int + switch { + case form == "host-port" && reachable: + ep = tcpEndpoint(testDoltServerPort) + case form == "host-port": + ep = tcpEndpoint(freeLoopbackPort(t)) + case reachable: + var sock string + sock, socketConnections = serveUnixProxy(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(testDoltServerPort))) + ep = unixEndpoint(sock) + default: + ep = unixEndpoint(externalInitSocketPath(t)) + } + + database := "provisioned" + if reachable { + database = uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + } + + args := append([]string{"init", "--init-if-missing", "--quiet", "--server", "--external"}, ep.flags...) + args = append(args, "-p", "prov", "--skip-hooks", "--skip-agents", "--database", database, f.repoDir) + pin := "" + if autoStart == "0" { + pin = "0" + } + + res := f.run(pin, args...) + + requireNoLaunch(t, res) + if !reachable { + if !res.failed { + t.Errorf("bd init succeeded against an unreachable endpoint %s:\n%s", ep.addr, res.output) + } + requireNothingListening(t, ep.network, ep.addr) + return + } + if res.failed { + t.Fatalf("bd init against the reachable server %s failed:\n%s", ep.addr, res.output) + } + requireDoltRootEmpty(t, f.repoDir) + data, err := os.ReadFile(filepath.Join(f.repoDir, ".beads", "metadata.json")) + if err != nil { + t.Fatalf("expected metadata.json after init: %v", err) + } + var meta configfile.Config + if err := json.Unmarshal(data, &meta); err != nil { + t.Fatalf("parse metadata.json: %v\n%s", err, data) + } + if form == "socket" { + if meta.DoltServerSocket != ep.addr { + t.Errorf("metadata.json dolt_server_socket = %q, want the socket init was given, %q", meta.DoltServerSocket, ep.addr) + } + if socketConnections() == 0 { + t.Errorf("bd never connected through the socket %s", ep.addr) + } + return + } + if meta.DoltServerHost != "127.0.0.1" || meta.DoltServerPort != testDoltServerPort { + t.Errorf("metadata.json names %s:%d, want the endpoint init was given, %s", meta.DoltServerHost, meta.DoltServerPort, ep.addr) + } +} diff --git a/cmd/bd/init_external_message_test.go b/cmd/bd/init_external_message_test.go new file mode 100644 index 000000000..2c06601df --- /dev/null +++ b/cmd/bd/init_external_message_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "errors" + "fmt" + "net" + "strings" + "testing" +) + +// TestExternalServerUnreachableMessage pins which failures get the --external +// wording. Only a failed dial may: the message claims the server could not be +// reached, so any other open failure (an identity mismatch, a handshake that +// was reset) must keep its own text instead of being reported as "externally +// managed". +func TestExternalServerUnreachableMessage(t *testing.T) { + refused := errors.New("connect: connection refused") + dial := &net.OpError{ + Op: "dial", + Net: "tcp", + Addr: &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 3307}, + Err: refused, + } + + tests := []struct { + name string + err error + wantEndpoint string // "" means the error is not rewritten + }{ + {"dial error", dial, "127.0.0.1:3307"}, + { + // The store wraps the dial error with %w and its own advice. + "dial error wrapped like the store does", + fmt.Errorf("Dolt server unreachable at 127.0.0.1:3307: %w\n\nThe Dolt server may not be running.", dial), + "127.0.0.1:3307", + }, + { + "unix socket dial error", + &net.OpError{Op: "dial", Net: "unix", Addr: &net.UnixAddr{Name: "/run/dolt.sock", Net: "unix"}, Err: refused}, + "/run/dolt.sock", + }, + { + // A name that does not resolve has no address to report. + "dial error without an address", + &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("no such host")}, + "the configured endpoint", + }, + {"read error after connecting", &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}, ""}, + {"not a network error", errors.New("project identity mismatch"), ""}, + } + // Every rewritten message says the server is declared external and that bd + // did not start one, and never advises `bd dolt start`. + requireDeclaredExternal := func(t *testing.T, got string, wants ...string) { + t.Helper() + for _, want := range append(wants, "externally managed", "did not start") { + if !strings.Contains(got, want) { + t.Errorf("message does not contain %q:\n%s", want, got) + } + } + if strings.Contains(got, "bd dolt start") { + t.Errorf("message advises `bd dolt start`:\n%s", got) + } + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := externalServerUnreachableMessage(tt.err) + if tt.wantEndpoint == "" { + if got != "" { + t.Fatalf("message = %q, want none for an error that is not a failed dial", got) + } + return + } + requireDeclaredExternal(t, got, tt.wantEndpoint) + }) + } + + // A dial to port 0 is an init that was never told a port: the address is a + // placeholder, not the server's endpoint, so the message must not offer it + // as one. The first line quotes the dialled address, so it is cut out + // before looking. + t.Run("dial error with no port named", func(t *testing.T) { + noPort := &net.OpError{ + Op: "dial", + Net: "tcp", + Addr: &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}, + Err: refused, + } + got := externalServerUnreachableMessage(noPort) + if got == "" { + t.Fatal("message = none, want the --external wording for a failed dial") + } + requireDeclaredExternal(t, got) + if strings.Contains(strings.ReplaceAll(got, noPort.Error(), ""), ":0") { + t.Errorf("message offers port 0 as the endpoint:\n%s", got) + } + }) +} diff --git a/cmd/bd/legacy_upgrade_guard.go b/cmd/bd/legacy_upgrade_guard.go index 53d680798..71ced0205 100644 --- a/cmd/bd/legacy_upgrade_guard.go +++ b/cmd/bd/legacy_upgrade_guard.go @@ -24,6 +24,19 @@ import ( // SQLite files, and the bounded local version witness; storage internals stay // behind the driver boundary. func guardLegacyUpgradeWorkspace(beadsDir string) error { + return guardLegacyUpgradeWorkspaceForInit(beadsDir, false) +} + +// guardLegacyUpgradeWorkspaceForInit is guardLegacyUpgradeWorkspace for a caller +// that has already resolved server intent from a source the workspace does not +// yet record: `bd init --server`, whose flag is the only place that choice lives +// until init has written it. The guard cannot see the flag, so initSelectsServer +// carries it in. It is consulted for exactly one thing, the empty-root admission +// below, and is deliberately not folded into the workspace's own server mode: +// that would also move a non-empty root with an unreadable witness onto the +// warn-and-open arm, and an embedded repository beside a non-empty .beads/dolt +// onto the refusing one. +func guardLegacyUpgradeWorkspaceForInit(beadsDir string, initSelectsServer bool) error { if beadsDir == "" { return nil } @@ -75,6 +88,15 @@ func guardLegacyUpgradeWorkspace(beadsDir string) error { // did not. return nil } + if initSelectsServer && !present && isEmptyDirectory(filepath.Join(beadsDir, "dolt")) { + // Init has already chosen a server, so this is the empty root the + // serverMode arm below admits for a workspace that records the choice: it + // holds no Dolt data a legacy release could have left, and bd makes it + // itself on use, so a retry after a first attempt that could not reach a + // server finds exactly this. A present witness, a root holding any entry, + // and every caller that does not pass the flag are decided as before. + return nil + } if serverMode { if !present && isEmptyDirectory(filepath.Join(beadsDir, "dolt")) { // An empty .beads/dolt holds no Dolt data, so no legacy release diff --git a/cmd/bd/legacy_upgrade_guard_init_flag_e2e_test.go b/cmd/bd/legacy_upgrade_guard_init_flag_e2e_test.go new file mode 100644 index 000000000..712842ffc --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_init_flag_e2e_test.go @@ -0,0 +1,87 @@ +//go:build cgo + +package main + +import ( + "os" + "path/filepath" + "slices" + "strconv" + "testing" +) + +// TestInitServerFlagAdmitsEmptyDoltRootWithoutConfig reproduces the retry shape +// with nothing on disk that records the server choice: an empty .beads/dolt and +// neither config.yaml nor metadata.json, so the --server flag is the only place +// the intent lives. The legacy guard used to refuse that init as a legacy Dolt +// workspace, so the workspace was never created and no bd command could repair +// it. +func TestInitServerFlagAdmitsEmptyDoltRootWithoutConfig(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, env, externalServerInitArgs(database)...) + if err != nil { + t.Fatalf("bd init --server over an empty .beads/dolt failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) +} + +// TestInitServerFlagRetryAfterFailedFirstAttempt replays the reported sequence: a +// first `bd init --server` that cannot reach a server fails after creating the +// local Dolt root, leaving an empty .beads/dolt and no witness, and the same +// command run again once the server is up must succeed instead of being refused +// as a legacy workspace over the root the first attempt made itself. +func TestInitServerFlagRetryAfterFailedFirstAttempt(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + // The first attempt has to fail without starting a server of its own: one + // that started and then failed would leave a populated .beads/dolt, which + // the guard refuses by design. externalServerTestEnv pins + // BEADS_DOLT_AUTO_START=0 for exactly that, which keeps the attempt + // deterministic even once init's separate --external handling stops trying + // to start a per-project server. Fail here, naming the reason, if a change to + // that helper drops it. + if !slices.Contains(env, "BEADS_DOLT_AUTO_START=0") { + t.Fatal("externalServerTestEnv must set BEADS_DOLT_AUTO_START=0: the failed first attempt must not start a server of its own") + } + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, env, + externalServerInitArgs(database, "--server-port", strconv.Itoa(freeLoopbackPort(t)))...) + if err == nil { + t.Fatalf("precondition: bd init --server against an unreachable server succeeded:\n%s", out) + } + // The retry only has something to fix if the failed attempt left exactly + // this. Any other leftover is a finding about init, not a reason to loosen + // what follows. + entries, readErr := os.ReadDir(filepath.Join(beadsDir, "dolt")) + if readErr != nil || len(entries) != 0 { + t.Fatalf("precondition: the failed first attempt should leave an empty .beads/dolt, found %d entries (err %v)", len(entries), readErr) + } + if _, statErr := os.Stat(filepath.Join(beadsDir, localVersionFile)); !os.IsNotExist(statErr) { + t.Fatalf("precondition: the failed first attempt should leave no %s (stat err %v)", localVersionFile, statErr) + } + + out, err = runExternalServerBD(t, repoDir, env, externalServerInitArgs(database)...) + if err != nil { + t.Fatalf("retrying bd init --server over the empty .beads/dolt the first attempt left failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) +} diff --git a/cmd/bd/legacy_upgrade_guard_init_flag_test.go b/cmd/bd/legacy_upgrade_guard_init_flag_test.go new file mode 100644 index 000000000..7fc5e9de5 --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_init_flag_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// TestLegacyUpgradeGuardForInitAdmitsEmptyDoltRoot pins the one thing init's own +// intent unlocks. `bd init --server` resolves server mode from the flag, a source +// the workspace does not record until init has written it. A retry over the empty +// .beads/dolt that a failed first attempt leaves behind therefore looks like an +// embedded workspace with a local Dolt root, and the guard refuses it. Told what +// init already knows, the guard gives that root the admission a config-selected +// server workspace gets. Asked without it, in either form, it still refuses, so +// the admission cannot pass vacuously. +func TestLegacyUpgradeGuardForInitAdmitsEmptyDoltRoot(t *testing.T) { + warnings := captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, "", "", "") + + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, true); err != nil { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, true) = %v, want nil", err) + } + if warnings.Len() != 0 { + t.Fatalf("guard warned about an absent witness: %q", warnings.String()) + } + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, false); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, false) = %v, want migration refusal", err) + } + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace(dir) = %v, want migration refusal", err) + } +} + +// TestLegacyUpgradeGuardForInitAdmissionStaysNarrow proves init's intent relaxes +// nothing beyond an empty, witness-less .beads/dolt. With it set, anything that +// could be legacy data still refuses, and the outcomes a broader reading of the +// flag would move (treating it as if the workspace itself selected server mode) +// stay where they are. +func TestLegacyUpgradeGuardForInitAdmissionStaysNarrow(t *testing.T) { + for _, entry := range []string{"beads", ".dolt", "stray-file", "symlink"} { + t.Run("root holding "+entry, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, "", "", "") + path := filepath.Join(beadsDir, "dolt", entry) + var err error + switch entry { + case "stray-file": + err = os.WriteFile(path, nil, 0o600) + case "symlink": + if err = os.Symlink(t.TempDir(), path); err != nil { + t.Skipf("symlink capability unavailable: %v", err) + } + default: + err = os.Mkdir(path, 0o700) + } + if err != nil { + t.Fatal(err) + } + + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, true); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, true) = %v, want migration refusal", err) + } + }) + } + + for _, version := range []string{"0.49.6", "0.62.0"} { + t.Run("pre-1.0 witness "+version, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, "", "", version) + + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, true); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, true) = %v, want migration refusal", err) + } + }) + } + + t.Run("blank witness over a non-empty root", func(t *testing.T) { + warnings := captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, "", "", "") + if err := os.Mkdir(filepath.Join(beadsDir, "dolt", "beads"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, localVersionFile), []byte("\n"), 0o600); err != nil { + t.Fatal(err) + } + + // A workspace that selects server mode itself warns and opens over an + // unreadable witness. Init's intent must not move this root onto that arm. + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, true); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, true) = %v, want migration refusal", err) + } + if warnings.Len() != 0 { + t.Fatalf("guard took the unreadable-witness arm: %q", warnings.String()) + } + }) + + t.Run("symlinked dolt root", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, "", "", "") + root := filepath.Join(beadsDir, "dolt") + if err := os.Remove(root); err != nil { + t.Fatal(err) + } + if err := os.Symlink(t.TempDir(), root); err != nil { + t.Skipf("symlink capability unavailable: %v", err) + } + + // hasLegacyDoltRoot does not follow a symlink, so the guard never takes + // such a root for a local Dolt root and it never reaches the empty-root + // admission. Init's intent has to leave its outcome exactly where every + // other caller finds it. + want := fmt.Sprint(guardLegacyUpgradeWorkspace(beadsDir)) + if got := fmt.Sprint(guardLegacyUpgradeWorkspaceForInit(beadsDir, true)); got != want { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, true) = %s, want %s (unchanged by init's intent)", got, want) + } + }) + + t.Run("embedded repository beside a non-empty root", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, "", "", "") + repo := filepath.Join(beadsDir, "embeddeddolt", "beads", ".dolt") + if err := os.MkdirAll(repo, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "repo-entry"), []byte("opaque"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "dolt", "stray-file"), nil, 0o600); err != nil { + t.Fatal(err) + } + + // An embedded repository admits the workspace before server intent is + // consulted at all. Folding init's intent into the workspace's own mode + // would send this one to the refusing arm. + for _, initSelectsServer := range []bool{false, true} { + if err := guardLegacyUpgradeWorkspaceForInit(beadsDir, initSelectsServer); err != nil { + t.Fatalf("guardLegacyUpgradeWorkspaceForInit(dir, %t) = %v, want nil", initSelectsServer, err) + } + } + }) +} From f6f122894f071fc485c49c9d489ca5710203a6aa Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 03:26:45 +0000 Subject: [PATCH 17/54] test(serve): before-parity pins for the httpapi startup event and ordinary-workspace bd serve (P13, P14) (#7679) * test(httpapi): pin the startup event's capabilities and database source TestStartupLines and TestStartupNamesTheDatabaseSource check the event=startup line by substring, and neither reads its capabilities field. A server could print one capability list and advertise another on GET /v0/beads/context, or a new kind of database source could relabel the line, and both tests would stay green. TestStartupEventPublishesTheAdvertisedCapabilitiesAndSource parses the line back into its fields and pins, for the provider and roles sources with and without the external-dependency policy: the exact field sequence, db=provider|roles, and a capabilities value equal to the handshake's list joined with commas. It is a before-parity pin: a future alternative source on the same server must leave this line unchanged for the Issue API's two sources. Test-only; no behavior change. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(serve): pin ordinary-workspace bd serve under BD_GRAPH_MODE=dependency bd reads no BD_GRAPH_MODE today. "dependency" names the format every existing workspace already has, so any later reader of that variable (a graph-mode serve dispatch, for example) has to keep serving an ordinary workspace through runServe exactly as it does now. These tests pin that before such a reader exists. Each test runs bd serve twice in one workspace, once with the variable unset and once with BD_GRAPH_MODE=dependency. It requires the event=startup and event=limits lines to match byte for byte apart from the ephemeral port, and the GET /v0/beads/context bodies to be equal. Each run is also checked against what runServe builds: the serveResolvedMode label, the db source, and a startup capabilities field equal to the handshake's list. Two runs that were both routed somewhere else therefore cannot pass by agreeing with each other. - TestServeOrdinaryWorkspaceIgnoresGraphModeDependency: in-process through rootCmd (PersistentPreRunE included), on the store source of a registered-backend workspace. Runs in bd_test. - TestServerModeServeIgnoresGraphModeDependency: a real bd on a `bd init --server` workspace, which uses the provider source. Gated on BEADS_TEST_PROXIED_SERVER like the other TestServerMode* tests, and placed in the proxied 34-shard manifest by gen_proxied_shard_manifest.py --write (incremental). The new file is listed in cmd/bd/BUILD.bazel's bd_test srcs. Test-only; no behavior change. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(serve): stop the in-process graph-mode pin hanging when serve fails serveInProcessSnapshot's cleanup decided whether the server was still running by looking at rootCancel. That holds for a clean run, which clears rootCancel in PersistentPostRunE. A run whose RunE returns an error skips PersistentPostRunE, so rootCancel stays set even though the main flow has already read the run's only result from done. The cleanup then called rootCancel and blocked on done for good, and the failure surfaced as a test-binary timeout panic that takes the whole bd_test shard with it instead of a FAIL for this test. The cleanup now keys on whether done has been read. A serve that binds, answers, and then returns an error (one way a future graph-serve dispatch could regress) now fails in about 5s with "bd serve returned , want a clean shutdown". A failure before done is read (a bad GET, say) still cancels the server and waits for it. Test-only; no behavior change. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 1 + cmd/bd/BUILD.bazel | 1 + cmd/bd/serve_graph_mode_env_test.go | 278 ++++++++++++++++++++ internal/httpapi/server_test.go | 119 +++++++++ 4 files changed, 399 insertions(+) create mode 100644 cmd/bd/serve_graph_mode_env_test.go diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 7c2e916b9..fca167226 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -157,6 +157,7 @@ 34 17 TestProxiedServerPrime 34 17 TestProxiedServerSQLDatabaseFlag 34 17 TestProxiedServerServeModeGate +34 17 TestServerModeServeIgnoresGraphModeDependency 34 18 TestProxiedServerClose3 34 18 TestProxiedServerCount diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 4a4e51b9d..8477c2d1a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1156,6 +1156,7 @@ go_test( "serve_delete_proxied_integration_test.go", "serve_dependencies_proxied_integration_test.go", "serve_edge_counts_proxied_integration_test.go", + "serve_graph_mode_env_test.go", "serve_inprocess_test.go", "serve_list_sort_proxied_integration_test.go", "serve_proxied_integration_test.go", diff --git a/cmd/bd/serve_graph_mode_env_test.go b/cmd/bd/serve_graph_mode_env_test.go new file mode 100644 index 000000000..fd61effd8 --- /dev/null +++ b/cmd/bd/serve_graph_mode_env_test.go @@ -0,0 +1,278 @@ +//go:build cgo + +package main + +import ( + "bytes" + "context" + "io" + "net/http" + "os" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/backends" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" +) + +// `bd serve` in an ordinary workspace is the same server whether or not +// BD_GRAPH_MODE=dependency is exported. +// +// bd reads no BD_GRAPH_MODE today. "dependency" is the value that names the +// format every existing workspace already has, so it is the one value a later +// reader of that variable (a graph-mode serve dispatch, say) must keep treating +// as "serve this workspace the way bd always has". These tests pin that before +// any such reader exists: with the variable exported, the run still has to +// reach runServe and build the same server. +// +// "The same server" is measured where an operator and a client measure it: the +// event=startup and event=limits lines, byte for byte apart from the ephemeral +// port, and the whole GET /v0/beads/context body. Each run is also checked on +// its own against what runServe builds (its mode label, its database source, +// the advertised capabilities), so two runs that were both routed somewhere +// else cannot pass by agreeing with each other. + +const serveGraphModeEnv = "BD_GRAPH_MODE" + +// TestServeOrdinaryWorkspaceIgnoresGraphModeDependency drives the whole root +// command in-process, PersistentPreRunE included, on a workspace served from +// the store source. In-process for the reason the registered-backend serve +// tests are: registration is Go wiring, so a spawned bd could not reach this +// arm. The server-mode twin below covers the provider source with a real bd. +func TestServeOrdinaryWorkspaceIgnoresGraphModeDependency(t *testing.T) { + const name = "serve-graph-mode-env" + + dir := t.TempDir() + initGitRepoAt(t, dir) + beadsDir := filepath.Join(dir, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + seedRegisteredBackendWorkspace(t, beadsDir) + backends.Register(name, backends.Backend{ + Open: func(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return embeddeddolt.Open(ctx, beadsDir, serveRegisteredDatabase, "main") + }, + OpenReadOnly: func(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return embeddeddolt.OpenReadOnly(ctx, beadsDir, serveRegisteredDatabase, "main") + }, + WorkspaceIsBeadsDir: true, + }) + t.Cleanup(func() { backends.Deregister(name) }) + if err := (&configfile.Config{Backend: name, DoltDatabase: serveRegisteredDatabase}).Save(beadsDir); err != nil { + t.Fatalf("save metadata.json: %v", err) + } + + // Unset rather than empty, so the baseline is a shell that never exported + // it. t.Setenv first so the original value comes back afterwards. + t.Setenv(serveGraphModeEnv, "") + if err := os.Unsetenv(serveGraphModeEnv); err != nil { + t.Fatalf("unset %s: %v", serveGraphModeEnv, err) + } + ordinary := serveInProcessSnapshot(t, dir, beadsDir) + + t.Setenv(serveGraphModeEnv, "dependency") + asserted := serveInProcessSnapshot(t, dir, beadsDir) + + mode := name + " (registered backend)" + assertRunServeBuiltIt(t, serveGraphModeEnv+" unset", ordinary, "roles", mode) + assertRunServeBuiltIt(t, serveGraphModeEnv+"=dependency", asserted, "roles", mode) + if asserted.context["backend"] != name { + t.Errorf("context backend = %v, want %q", asserted.context["backend"], name) + } + assertSameServer(t, ordinary, asserted) +} + +// TestServerModeServeIgnoresGraphModeDependency is the same pin for a +// `bd init --server` workspace, the topology an ordinary deployment serves +// from: the provider source, which serve builds itself. It runs a real bd, +// because that arm fronts the server through a proxy child, and a proxy child +// cannot be spawned from inside the test binary. +func TestServerModeServeIgnoresGraphModeDependency(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newServerModeProject(t, bd, "srvgm") + + ordinaryEnv := envWithout(p.env, serveGraphModeEnv) + assertedEnv := append(slices.Clone(ordinaryEnv), serveGraphModeEnv+"=dependency") + + ordinary := serveProcessSnapshot(t, startServe(t, bd, p.dir, ordinaryEnv)) + asserted := serveProcessSnapshot(t, startServe(t, bd, p.dir, assertedEnv)) + + const mode = "server (external dolt)" + assertRunServeBuiltIt(t, serveGraphModeEnv+" unset", ordinary, "provider", mode) + assertRunServeBuiltIt(t, serveGraphModeEnv+"=dependency", asserted, "provider", mode) + if asserted.context["dolt_mode"] != "server" || asserted.context["database"] != p.database { + t.Errorf("context dolt_mode/database = %v/%v, want server/%s", + asserted.context["dolt_mode"], asserted.context["database"], p.database) + } + assertSameServer(t, ordinary, asserted) +} + +// serveSnapshot is what one bd serve run shows before any operation: its two +// startup events (from "event=" on, without the log prefix) and its handshake. +type serveSnapshot struct { + addr string + startup string + limits string + context map[string]any +} + +// serveInProcessSnapshot runs `bd serve` through rootCmd, reads the handshake, +// and shuts it down the way an operator does. +func serveInProcessSnapshot(t *testing.T, dir, beadsDir string) serveSnapshot { + t.Helper() + stopStderr := captureStderrText(t) + addr, done := startServeInProcess(t, dir, beadsDir) + // Stops a server that a failure below left running. Keyed on whether done + // has been read, not on rootCancel: a run that returns an error skips + // PersistentPostRunE and leaves rootCancel set, and waiting on done again + // after reading its only result would hang the test binary. + drained := false + t.Cleanup(func() { + if !drained && rootCancel != nil { + rootCancel() + <-done + } + }) + body := getJSON(t, "http://"+addr+"/v0/beads/context") + if rootCancel == nil { + t.Fatal("the root command published no cancel function; nothing here can stop the server") + } + rootCancel() + err := <-done + drained = true + if err != nil { + t.Fatalf("bd serve returned %v, want a clean shutdown", err) + } + log := stopStderr() + return serveSnapshot{ + addr: addr, + startup: serveEventLine(t, log, "startup"), + limits: serveEventLine(t, log, "limits"), + context: body, + } +} + +// serveProcessSnapshot reads a spawned serve's startup and handshake, then +// stops it with SIGTERM. +func serveProcessSnapshot(t *testing.T, sp *serveProcess) serveSnapshot { + t.Helper() + startup := sp.awaitLogLine(t, "event=startup") + limits := sp.awaitLogLine(t, "event=limits") + status, body, _ := sp.get(t, "/v0/beads/context") + if status != http.StatusOK { + t.Fatalf("GET /v0/beads/context status = %d, want 200: %v", status, body) + } + sp.shutdown(t) + return serveSnapshot{ + addr: sp.addr, + startup: serveEventLine(t, startup, "startup"), + limits: serveEventLine(t, limits, "limits"), + context: body, + } +} + +// serveEventLine returns the named event's line from "event=" on. +func serveEventLine(t *testing.T, log, name string) string { + t.Helper() + for _, line := range strings.Split(log, "\n") { + if i := strings.Index(line, "event="+name+" "); i >= 0 { + return line[i:] + } + } + t.Fatalf("no event=%s line in:\n%s", name, log) + return "" +} + +// assertRunServeBuiltIt checks one run against the server runServe builds: +// serveResolvedMode's label, the Issue API's database source, and a +// capabilities field equal to the handshake's. +func assertRunServeBuiltIt(t *testing.T, run string, s serveSnapshot, source, mode string) { + t.Helper() + assertServedCapabilities(t, s.context) + if s.context["api_version"] != "v0" { + t.Errorf("%s: context api_version = %v, want v0", run, s.context["api_version"]) + } + caps, _ := s.context["capabilities"].([]any) + tokens := make([]string, 0, len(caps)) + for _, c := range caps { + token, _ := c.(string) + tokens = append(tokens, token) + } + for _, field := range []string{ + "addr=" + s.addr, + "mode=" + strconv.Quote(mode), + "db=" + source, + "capabilities=" + strings.Join(tokens, ","), + } { + if !strings.Contains(s.startup+" ", " "+field+" ") { + t.Errorf("%s: startup line is missing %s:\n%s", run, field, s.startup) + } + } +} + +// assertSameServer compares two runs on everything but the ephemeral port. +func assertSameServer(t *testing.T, ordinary, asserted serveSnapshot) { + t.Helper() + normalize := func(s serveSnapshot) string { + return strings.Replace(s.startup, " addr="+s.addr+" ", " addr= ", 1) + } + if got, want := normalize(asserted), normalize(ordinary); got != want { + t.Errorf("%s=dependency changed the startup line:\n got: %s\nwant: %s", serveGraphModeEnv, got, want) + } + if asserted.limits != ordinary.limits { + t.Errorf("%s=dependency changed the limits line:\n got: %s\nwant: %s", serveGraphModeEnv, asserted.limits, ordinary.limits) + } + if !reflect.DeepEqual(asserted.context, ordinary.context) { + t.Errorf("%s=dependency changed GET /v0/beads/context:\n got: %v\nwant: %v", serveGraphModeEnv, asserted.context, ordinary.context) + } +} + +// captureStderrText redirects os.Stderr until the returned function is called, +// and returns everything written meanwhile. The restore also runs at cleanup, +// so a t.Fatal partway through cannot leak the redirect into later tests; a +// failed test logs what was captured. +func captureStderrText(t *testing.T) func() string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + orig := os.Stderr + os.Stderr = w + + var buf bytes.Buffer + copied := make(chan struct{}) + go func() { + defer close(copied) + _, _ = io.Copy(&buf, r) + }() + + var once sync.Once + var out string + stop := func() string { + once.Do(func() { + os.Stderr = orig + _ = w.Close() + <-copied + _ = r.Close() + out = buf.String() + }) + return out + } + t.Cleanup(func() { + if text := stop(); t.Failed() { + t.Logf("captured stderr:\n%s", text) + } + }) + return stop +} diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index ba2105508..54f43e91c 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "testing" @@ -1275,6 +1276,124 @@ func TestStartupLines(t *testing.T) { } } +// TestStartupEventPublishesTheAdvertisedCapabilitiesAndSource pins the whole +// event=startup line of a provider- or roles-backed server: its field sequence, +// its database-source label, and a capabilities field that is the handshake's +// list verbatim. +// +// TestStartupLines and TestStartupNamesTheDatabaseSource check fields by +// substring, and neither reads capabilities at all. So a server could print one +// list and advertise another, or a new kind of database source could relabel +// this line, and both would stay green. The line is how an operator learns +// what a server answers before any client connects, so it is pinned exactly: +// changing it should take a deliberate edit to this test, not happen as a side +// effect of adding a source. +func TestStartupEventPublishesTheAdvertisedCapabilitiesAndSource(t *testing.T) { + workspace := domain.ContextInfo{RepoRoot: "/w/repo", BeadsDir: "/w/repo/.beads", Database: "beads"} + for _, tc := range []struct { + name string + cfg Config + source string + }{ + {"provider", Config{Provider: &tunableProvider{}}, "provider"}, + {"provider with the external-dependency policy", Config{Provider: &tunableProvider{}, ExternalDependencyPolicy: true}, "provider"}, + {"roles", rolesConfig(Config{}), "roles"}, + {"roles with the external-dependency policy", rolesConfig(Config{ExternalDependencyPolicy: true}), "roles"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.Mode = "server (external dolt)" + cfg.Workspace = workspace + ts := newTestServer(t, cfg) + + var advertised []string + caps, _ := decodeBody(t, ts.get(t, "/v0/beads/context"))["capabilities"].([]any) + for _, c := range caps { + advertised = append(advertised, c.(string)) + } + if len(advertised) == 0 { + t.Fatal("the handshake advertises no capabilities; comparing the startup line against it would pass for an empty one") + } + // The conditional token is in the list exactly when the case asks + // for it, so the policy rows really cover a list that differs. + if got := slices.Contains(advertised, CapExternalDependencies); got != cfg.ExternalDependencyPolicy { + t.Fatalf("handshake advertises %s = %v, want %v", CapExternalDependencies, got, cfg.ExternalDependencyPolicy) + } + + line := findLogLine(t, ts.stderr.String(), "event=startup") + keys, fields := eventFields(t, line) + want := []struct{ key, value string }{ + {"event", "startup"}, + {"addr", ts.Addr()}, + {"mode", cfg.Mode}, + {"db", tc.source}, + {"workspace", workspace.RepoRoot}, + {"beads_dir", workspace.BeadsDir}, + {"database", workspace.Database}, + {"host_allowlist", "127.0.0.1,::1,localhost"}, + {"capabilities", strings.Join(advertised, ",")}, + {"auth", "none"}, + } + var wantKeys []string + for _, w := range want { + wantKeys = append(wantKeys, w.key) + if got := fields[w.key]; got != w.value { + t.Errorf("startup %s = %q, want %q", w.key, got, w.value) + } + } + if !slices.Equal(keys, wantKeys) { + t.Errorf("startup fields = %v, want %v\n%s", keys, wantKeys, line) + } + }) + } +} + +// eventFields splits one event line back into its fields, in order: the +// inverse of Server.event, where every value is a bare token or a Go-quoted +// string (logValue). The log prefix before "event=" is dropped. +func eventFields(t *testing.T, line string) ([]string, map[string]string) { + t.Helper() + start := strings.Index(line, "event=") + if start < 0 { + t.Fatalf("not an event line: %q", line) + } + rest := line[start:] + var keys []string + fields := map[string]string{} + for rest != "" { + key, value, ok := strings.Cut(rest, "=") + if !ok || key == "" || strings.ContainsAny(key, ` "`) { + t.Fatalf("malformed field at %q in %q", rest, line) + } + if strings.HasPrefix(value, `"`) { + quoted, err := strconv.QuotedPrefix(value) + if err != nil { + t.Fatalf("malformed quoted value at %q in %q: %v", value, line, err) + } + unquoted, err := strconv.Unquote(quoted) + if err != nil { + t.Fatalf("unquote %s: %v", quoted, err) + } + rest = value[len(quoted):] + value = unquoted + if rest != "" { + if rest[0] != ' ' { + t.Fatalf("field %s is not followed by a space in %q", key, line) + } + rest = rest[1:] + } + } else { + value, rest, _ = strings.Cut(value, " ") + } + if _, dup := fields[key]; dup { + t.Fatalf("field %s appears twice in %q", key, line) + } + keys = append(keys, key) + fields[key] = value + } + return keys, fields +} + // TestPoolLimitsUnavailableIsAnnounced: running unbounded is a decision, so it // must not be a silent one. func TestPoolLimitsUnavailableIsAnnounced(t *testing.T) { From f142cc2ffdece62ae5830a7f436e4949a48c79fe Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 03:41:08 +0000 Subject: [PATCH 18/54] test(cli): before-parity pins for --format json, ordinary --if-revision, and version/config/history/as-of shapes (P7, P8, P10) (#7676) * test(cli): pin --format json == --json on show, list, ready and memories Before-parity pin P7 (gap analysis 2026-10-17, section 5a). The hidden root --format flag is an alias for --json (GH#2612). It is a root persistent flag, so a subcommand that registers a --format of its own shadows it (the GH#6278 hazard): `bd list` already does and passes `json` through by hand, and a graph-preview fork that added a local --format to `bd show` made `bd show ID --format json` exit nonzero. TestEmbeddedFormatJSONAliasMatchesJSON runs show, list, ready and memories with --json and then with `--format json`, `--format=json` and a leading `bd --format json `, and requires exit 0 and byte-identical stdout. It adds bdRunStreams, a small helper returning stdout, stderr and the exit status separately, which the following pins reuse. The new TestEmbedded* function is listed in the bd_test srcs and placed in the 100-shard embedded cmd manifest by gen_embedded_cmd_shard_manifest.py --write (incremental). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(cli): pin the ordinary --if-revision compare-and-swap contract Before-parity pin P8 (gap analysis 2026-10-17, section 5a). --if-revision is the decimal int64 compare-and-swap merged in #7203, and gascity's bdstore_conditional.go classifier reads both its exit status and its machine body. A second, opaque-token meaning for the same flag in another kind of workspace must leave the ordinary one unchanged. TestEmbeddedIfRevisionOrdinaryCAS pins, through the real binary: - a stale decimal token on update, close, assign and delete (preview and --force) exits 13, prints nothing on stdout, writes nothing, and ends stderr with a precondition_failed body naming the id and the expected and current revisions as exact JSON integers; - a non-decimal token exits 1 with the "invalid --if-revision" refusal in text and --json modes, never 13, and writes nothing; - the current token applies on update, close, assign and delete. It is the ordinary-workspace half of TestGraphPreviewIfRevisionOutsideLinkModeIsUpstreamCAS from the donnabox/codex/preview3-source-catchup-46aa branch (added in 3d16d826be, revised in 1640940fa9, 3ff2fe73ce, 55dbb61156, 653369305a and 97446d1f08), ported onto this package's embedded helpers. The port drops the graph-only forget refusal, extends the stale and non-decimal cases from update/delete to close and assign, and compares revisions as json.Number rather than float64, since they are int64 values past 2^53. It is named TestEmbedded* so the bazel-embedded lane, which discovers `^func TestEmbedded` in *_embedded_test.go, runs it; the existing TestIfRevisionMatchAndMismatch is not discovered there. The file is listed in the bd_test srcs and placed in the 100-shard embedded cmd manifest by gen_embedded_cmd_shard_manifest.py --write (incremental). Co-authored-by: Jim Wordelman Co-authored-by: Donna Box Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(cli): pin version, config set, history and show --as-of shapes Before-parity pin P10 (gap analysis 2026-10-17, section 5a): the store-less and history-adjacent commands a later versioned-history or graph slice could change by accident. - TestEmbeddedVersionOutputShape: `bd version` with no arguments exits 0 and prints one "bd version (...)" line, in a workspace and outside one; --json is one object with version, build and schema_version (plus commit and branch when known) agreeing with the text line, and --format json prints the same document; an argument exits 1 and points at `bd versions ` (#6661), on stderr or as the --json error, through the real binary rather than RunE. - TestEmbeddedConfigSetOtherKeysSkipTheVersionedHistoryScan: in a store holding a row the pre-enable scan refuses over, `bd config set` and `bd config set-many` of another key with the value "true" exit 0, store it, print no refusal and leave the switch off; the switch itself is still refused afterwards, as a control. - TestEmbeddedHistoryAndShowAsOfReadDoltCommits: with versioned history recording, `bd history --json` is still the Dolt-commit history (an array of CommitHash/Committer/CommitDate/Issue entries with full 32-character Dolt hashes, no recorded-version fields, and the same hashes in the text output), and `bd show --as-of ` with a hash from it is the Dolt AS OF read in show's JSON shape, not the R7.1 as-of read over issue_versions; `--as-of main` reads the branch head. The three TestEmbedded* functions are placed in the 100-shard embedded cmd manifest by gen_embedded_cmd_shard_manifest.py --write (incremental). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Jim Wordelman Co-authored-by: Donna Box --- .github/scripts/embedded-cmd-test-shards.txt | 5 + cmd/bd/BUILD.bazel | 2 + cmd/bd/format_json_alias_embedded_test.go | 102 +++++++++++ cmd/bd/history_embedded_test.go | 127 +++++++++++++ .../if_revision_ordinary_cas_embedded_test.go | 168 ++++++++++++++++++ cmd/bd/version_embedded_test.go | 95 ++++++++++ ...ioned_history_switch_scan_embedded_test.go | 50 ++++++ 7 files changed, 549 insertions(+) create mode 100644 cmd/bd/format_json_alias_embedded_test.go create mode 100644 cmd/bd/if_revision_ordinary_cas_embedded_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 983c57438..e05f241a2 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -123,13 +123,18 @@ 100 49 TestEmbeddedInitAPushConsentAndRemoteBootstrap 100 50 TestEmbeddedDiffConcurrent +100 50 TestEmbeddedVersionOutputShape +100 51 TestEmbeddedHistoryAndShowAsOfReadDoltCommits 100 51 TestEmbeddedInitGitBootstrapRoutingFreshQuiet +100 52 TestEmbeddedConfigSetOtherKeysSkipTheVersionedHistoryScan 100 52 TestEmbeddedSwarmConcurrent +100 53 TestEmbeddedIfRevisionOrdinaryCAS 100 53 TestEmbeddedInitSelectedExcludeRoutingStealthDecoyQuiet +100 54 TestEmbeddedFormatJSONAliasMatchesJSON 100 54 TestEmbeddedSearchConcurrent 100 55 TestEmbeddedUndefer diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 8477c2d1a..f16e65cbb 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -804,6 +804,7 @@ go_test( "find_duplicates_test.go", "flags_test.go", "flatten_embedded_test.go", + "format_json_alias_embedded_test.go", "formula_fixture_proxied_test.go", "formula_help_test.go", "formula_primitives_test.go", @@ -865,6 +866,7 @@ go_test( "identity_preview_refusal_test.go", "if_revision_embedded_test.go", "if_revision_gc_matcher_embedded_test.go", + "if_revision_ordinary_cas_embedded_test.go", "if_revision_proxied_close_delete_test.go", "if_revision_proxied_integration_test.go", "if_revision_reassign_fence_embedded_test.go", diff --git a/cmd/bd/format_json_alias_embedded_test.go b/cmd/bd/format_json_alias_embedded_test.go new file mode 100644 index 000000000..3e163c870 --- /dev/null +++ b/cmd/bd/format_json_alias_embedded_test.go @@ -0,0 +1,102 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "os/exec" + "slices" + "strings" + "testing" +) + +// bdRunStreams runs one bd process in dir with the hermetic test environment +// and returns stdout, stderr and the exit status separately. A nonzero exit is +// returned, not fatal, so a caller can pin the status itself; only a process +// that could not be started fails the test here. +func bdRunStreams(t *testing.T, bd, dir string, args ...string) (stdout, stderr string, exit int) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + var outBuf, errBuf bytes.Buffer + cmd.Stdout, cmd.Stderr = &outBuf, &errBuf + if err := cmd.Run(); err != nil { + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Fatalf("bd %s did not run: %v", strings.Join(args, " "), err) + } + return outBuf.String(), errBuf.String(), ee.ExitCode() + } + return outBuf.String(), errBuf.String(), 0 +} + +// TestEmbeddedFormatJSONAliasMatchesJSON pins the hidden root --format alias +// (GH#2612): on show, list, ready and memories, `--format json` prints exactly +// what `--json` prints, byte for byte, and exits 0 like it. +// +// The alias is a root persistent flag, so any subcommand that registers a +// --format of its own shadows it (the GH#6278 hazard). `bd list` already has +// its own --format and passes `json` through by hand (list_input.go). A +// local --format added to show, ready or memories for another output +// format would turn `bd show ID --format json` into a refusal or into a +// different document. Both are regressions this test catches: it compares +// the whole of stdout, not a parse of it. +// +// Each command is checked in three spellings: `--format json` and +// `--format=json` after the subcommand, and `--format json` before it, where +// the flag is plainly the root's. +func TestEmbeddedFormatJSONAliasMatchesJSON(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "fj") + + id := bdCreateSilent(t, bd, dir, "Format alias parity", "--type", "task", "--description", "read through --format json") + bdRunOK(t, bd, dir, "remember", "the format alias is --json", "--key", "fj-alias") + + for _, tc := range []struct { + name string + args []string + // mention is a value the --json document must contain, so two empty + // documents cannot match each other. + mention string + }{ + {"show", []string{"show", id}, id}, + {"list", []string{"list"}, id}, + {"ready", []string{"ready"}, id}, + {"memories", []string{"memories"}, "fj-alias"}, + } { + t.Run(tc.name, func(t *testing.T) { + wantArgs := slices.Concat(tc.args, []string{"--json"}) + want, wantErr, exit := bdRunStreams(t, bd, dir, wantArgs...) + if exit != 0 { + t.Fatalf("bd %s exited %d\nstdout:\n%s\nstderr:\n%s", strings.Join(wantArgs, " "), exit, want, wantErr) + } + if !json.Valid([]byte(want)) || !strings.Contains(want, tc.mention) { + t.Fatalf("bd %s did not print one JSON document naming %s:\n%s", strings.Join(wantArgs, " "), tc.mention, want) + } + + for _, args := range [][]string{ + slices.Concat(tc.args, []string{"--format", "json"}), + slices.Concat(tc.args, []string{"--format=json"}), + slices.Concat([]string{"--format", "json"}, tc.args), + } { + got, gotErr, exit := bdRunStreams(t, bd, dir, args...) + if exit != 0 { + t.Errorf("bd %s exited %d, want 0 like bd %s\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), exit, strings.Join(wantArgs, " "), got, gotErr) + continue + } + if got != want { + t.Errorf("bd %s printed a different document from bd %s\n--format:\n%s\n--json:\n%s", strings.Join(args, " "), strings.Join(wantArgs, " "), got, want) + } + } + }) + } +} diff --git a/cmd/bd/history_embedded_test.go b/cmd/bd/history_embedded_test.go index 5c7910c20..ab351a1dd 100644 --- a/cmd/bd/history_embedded_test.go +++ b/cmd/bd/history_embedded_test.go @@ -7,6 +7,8 @@ import ( "fmt" "os" "os/exec" + "regexp" + "sort" "strings" "sync" "testing" @@ -403,3 +405,128 @@ func TestEmbeddedHistoryConcurrent(t *testing.T) { } } } + +// doltCommitHashPattern is a full Dolt commit hash: 32 characters of Dolt's +// base32 alphabet. It is what `bd show --as-of` takes, and a prefix is not. +var doltCommitHashPattern = regexp.MustCompile(`^[0-9a-v]{32}$`) + +// issueVersionOnlyKeys are fields of a recorded version (`bd versions --json`, +// storage.IssueVersion) that neither the Dolt-commit history nor the Dolt AS OF +// read has. +var issueVersionOnlyKeys = []string{"local_revision", "epoch", "change_actor", "change_at", "attribution_status", "removed_restriction", "state_bytes"} + +// TestEmbeddedHistoryAndShowAsOfReadDoltCommits pins what `bd history ` and +// `bd show --as-of ` read in an ordinary workspace whose store records +// versioned history, so both planes hold the same issue: +// - `bd history --json` is the Dolt-commit history: an array, newest first, +// of {CommitHash, Committer, CommitDate, Issue} entries, each CommitHash a full +// Dolt commit hash and each Issue the row as that commit left it. No field of a +// recorded version appears, and the text output names the same full hashes. +// - `bd show --as-of `, with a hash from `bd history`, is the Dolt AS OF +// read: the issue as that commit left it, in show's JSON shape (an array of +// issues), not a recorded version. A branch name is a ref too: --as-of main +// reads the branch head. +// +// Recorded versions are a separate plane with its own command (`bd versions`) and +// its own as-of read (R7.1, over issue_versions), which takes an instant or a +// version address and not a Dolt ref. Recording them must change neither answer. +func TestEmbeddedHistoryAndShowAsOfReadDoltCommits(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "hd") + bdConfig(t, bd, dir, "set", "versioned-history.enabled", "true") + + id := bdCreateSilent(t, bd, dir, "Before title", "--type", "task") + bdUpdate(t, bd, dir, id, "--title", "After title") + + // Both planes hold this issue: versions were recorded for it. + var recorded struct { + Recording bool `json:"recording"` + Versions []json.RawMessage `json:"versions"` + } + if out, stderr, exit := bdRunStreams(t, bd, dir, "versions", id, "--json"); exit != 0 || json.Unmarshal([]byte(out), &recorded) != nil || !recorded.Recording || len(recorded.Versions) == 0 { + t.Fatalf("bd versions %s --json (exit %d) did not report recorded versions, so this test would not tell the planes apart\nstdout:\n%s\nstderr:\n%s", id, exit, out, stderr) + } + + historyJSON := bdHistory(t, bd, dir, id, "--json") + var entries []map[string]json.RawMessage + if err := json.Unmarshal([]byte(historyJSON), &entries); err != nil { + t.Fatalf("bd history %s --json is not a JSON array: %v\n%s", id, err, historyJSON) + } + if len(entries) < 2 { + t.Fatalf("bd history %s --json has %d entries, want one for each of the create and the update:\n%s", id, len(entries), historyJSON) + } + type snapshot struct { + ID string `json:"id"` + Title string `json:"title"` + } + hashes := make([]string, len(entries)) + titles := make([]string, len(entries)) + for i, entry := range entries { + keys := make([]string, 0, len(entry)) + for key := range entry { + keys = append(keys, key) + } + sort.Strings(keys) + if got := strings.Join(keys, ","); got != "CommitDate,CommitHash,Committer,Issue" { + t.Errorf("bd history entry %d has keys %s, want the Dolt-commit entry CommitDate,CommitHash,Committer,Issue", i, got) + } + if err := json.Unmarshal(entry["CommitHash"], &hashes[i]); err != nil || !doltCommitHashPattern.MatchString(hashes[i]) { + t.Errorf("bd history entry %d CommitHash %s is not a full Dolt commit hash", i, entry["CommitHash"]) + } + var issue snapshot + if err := json.Unmarshal(entry["Issue"], &issue); err != nil || issue.ID != id { + t.Errorf("bd history entry %d Issue %s is not a snapshot of %s", i, entry["Issue"], id) + } + titles[i] = issue.Title + } + if titles[0] != "After title" || titles[len(titles)-1] != "Before title" { + t.Fatalf("bd history %s titles newest first are %q, want %q first and %q last", id, titles, "After title", "Before title") + } + for _, key := range issueVersionOnlyKeys { + if strings.Contains(historyJSON, `"`+key+`"`) { + t.Errorf("bd history %s --json carries %q, a field of a recorded version:\n%s", id, key, historyJSON) + } + } + text := bdHistory(t, bd, dir, id) + for _, hash := range hashes { + if !strings.Contains(text, hash) { + t.Errorf("bd history %s text output does not name the full commit hash %s:\n%s", id, hash, text) + } + } + + // show --as-of reads the Dolt commit the history named. + first := hashes[len(hashes)-1] + asOfJSON := func(ref string) snapshot { + t.Helper() + out, stderr, exit := bdRunStreams(t, bd, dir, "show", id, "--as-of", ref, "--json") + var shown []map[string]json.RawMessage + if exit != 0 || json.Unmarshal([]byte(out), &shown) != nil || len(shown) != 1 { + t.Fatalf("bd show %s --as-of %s --json exited %d, want an array of one issue\nstdout:\n%s\nstderr:\n%s", id, ref, exit, out, stderr) + } + for _, key := range issueVersionOnlyKeys { + if _, ok := shown[0][key]; ok { + t.Errorf("bd show %s --as-of %s --json carries %q, a field of a recorded version:\n%s", id, ref, key, out) + } + } + var issues []snapshot + if err := json.Unmarshal([]byte(out), &issues); err != nil { + t.Fatalf("bd show %s --as-of %s --json: %v\n%s", id, ref, err, out) + } + return issues[0] + } + if got := asOfJSON(first); got.ID != id || got.Title != "Before title" { + t.Errorf("bd show %s --as-of %s (its first commit) read %+v, want the issue titled %q", id, first, got, "Before title") + } + if got := asOfJSON("main"); got.ID != id || got.Title != "After title" { + t.Errorf("bd show %s --as-of main read %+v, want the branch head's title %q", id, got, "After title") + } + human := bdShowRaw(t, bd, dir, id, "--as-of", first) + if !strings.Contains(human, "as of "+first) || !strings.Contains(human, "Before title") || strings.Contains(human, "After title") { + t.Errorf("bd show %s --as-of %s text output does not show the first commit's issue under that ref:\n%s", id, first, human) + } +} diff --git a/cmd/bd/if_revision_ordinary_cas_embedded_test.go b/cmd/bd/if_revision_ordinary_cas_embedded_test.go new file mode 100644 index 000000000..c51b058fa --- /dev/null +++ b/cmd/bd/if_revision_ordinary_cas_embedded_test.go @@ -0,0 +1,168 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "slices" + "strconv" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// ifRevisionStaleExit is a literal on purpose: the contract with gascity's +// bdstore_conditional.go classifier is the number 13, not whatever name the +// constant for it (ExitGuardMismatch) happens to have. +const ifRevisionStaleExit = 13 + +// decodeIfRevisionBodyExact decodes the flat machine body a refused +// --if-revision write puts on the last line of stderr, keeping numbers as +// json.Number. Revisions are int64 values well past 2^53, so comparing them +// as float64 would accept a neighbouring revision; a json.Number also proves +// the field is a JSON number and not a string. +func decodeIfRevisionBodyExact(t *testing.T, line string) map[string]any { + t.Helper() + dec := json.NewDecoder(strings.NewReader(line)) + dec.UseNumber() + var body map[string]any + if err := dec.Decode(&body); err != nil { + t.Fatalf("decode the --if-revision refusal body: %v\nline: %s", err, line) + } + return body +} + +// TestEmbeddedIfRevisionOrdinaryCAS pins the ordinary --if-revision +// compare-and-swap (#7203) end to end, as a gascity-style caller sees it: +// +// - a stale decimal token on update, close, assign and delete (preview and +// --force) exits 13 with a precondition_failed body on the last line of +// stderr naming the id and both revisions as JSON integers, prints +// nothing on stdout and writes nothing; +// - a token that is not a decimal exits 1 with the "invalid --if-revision" +// refusal, never 13, and writes nothing; +// - the current token applies on update, close, assign and delete, and +// each write that leaves the issue in place moves its revision on. +// +// A second meaning for the same flag (an opaque token in some other kind of +// workspace) must leave all of this unchanged in an ordinary one. This is +// the ordinary-workspace half of the fork's +// TestGraphPreviewIfRevisionOutsideLinkModeIsUpstreamCAS, ported onto this +// package's embedded helpers and extended from update/delete to close and +// assign. +func TestEmbeddedIfRevisionOrdinaryCAS(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ifrev") + + id := bdCreateSilent(t, bd, dir, "Stale guard", "--type", "task", "--priority", "2") + stale := bdShowRevision(t, bd, dir, id) + bdRunOK(t, bd, dir, "update", id, "--notes", "bump", "--json") + current := bdShowRevision(t, bd, dir, id) + if current == stale { + t.Fatal("an unguarded update did not advance the revision") + } + staleToken := strconv.FormatInt(stale, 10) + + for _, tc := range []struct { + name string + args []string + }{ + {"update", []string{"update", id, "--priority", "3"}}, + {"close", []string{"close", id}}, + {"assign", []string{"assign", id, "worker"}}, + {"delete preview", []string{"delete", id}}, + {"delete force", []string{"delete", id, "--force"}}, + } { + args := slices.Concat(tc.args, []string{"--if-revision", staleToken, "--json"}) + stdout, stderr, exit := bdRunStreams(t, bd, dir, args...) + if exit != ifRevisionStaleExit { + t.Fatalf("stale --if-revision %s exited %d, want %d\nstdout:\n%s\nstderr:\n%s", tc.name, exit, ifRevisionStaleExit, stdout, stderr) + } + if stdout != "" { + t.Errorf("stale --if-revision %s printed on stdout, want nothing:\n%s", tc.name, stdout) + } + body := decodeIfRevisionBodyExact(t, lastJSONLine(t, stderr)) + if body["code"] != ifRevisionCodePreconditionFailed || body["id"] != id { + t.Errorf("stale --if-revision %s reported %v, want code %q for %s", tc.name, body, ifRevisionCodePreconditionFailed, id) + } + if got := body["expected_revision"]; got != json.Number(staleToken) { + t.Errorf("stale --if-revision %s reported expected_revision %#v, want the JSON integer %s", tc.name, got, staleToken) + } + if got := body["current_revision"]; got != json.Number(strconv.FormatInt(current, 10)) { + t.Errorf("stale --if-revision %s reported current_revision %#v, want the JSON integer %d", tc.name, got, current) + } + if msg, _ := body["error"].(string); !strings.Contains(msg, "precondition failed") { + t.Errorf("stale --if-revision %s error text %q lacks \"precondition failed\"", tc.name, msg) + } + if got := bdShowRevision(t, bd, dir, id); got != current { + t.Fatalf("stale --if-revision %s wrote: revision %d -> %d", tc.name, current, got) + } + } + + // A token that is not a decimal is a typo refused before any write, with + // exit 1 and not the guard's 13, in both output modes. + for _, base := range [][]string{ + {"update", id, "--priority", "3"}, + {"close", id}, + {"assign", id, "worker"}, + {"delete", id}, + {"delete", id, "--force"}, + } { + for _, mode := range [][]string{nil, {"--json"}} { + args := slices.Concat(base, []string{"--if-revision", "observed"}, mode) + stdout, stderr, exit := bdRunStreams(t, bd, dir, args...) + out := stdout + stderr + // The text refusal quotes the token; the --json one carries the + // same text JSON-escaped. + refused := strings.Contains(out, `invalid --if-revision "observed"`) || + strings.Contains(out, `invalid --if-revision \"observed\"`) + if exit != 1 || !refused { + t.Errorf("bd %s exited %d, want exit 1 refusing the non-decimal token\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), exit, stdout, stderr) + } + if strings.Contains(out, ifRevisionCodePreconditionFailed) { + t.Errorf("bd %s reported a stale revision for a token that is not one:\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), stdout, stderr) + } + } + } + if got := bdShowRevision(t, bd, dir, id); got != current { + t.Fatalf("a refused non-decimal token wrote: revision %d -> %d", current, got) + } + + // The current token applies, on each verb in turn. + revision := current + guarded := func(target string, args ...string) { + t.Helper() + bdRunOK(t, bd, dir, slices.Concat(args, []string{"--if-revision", strconv.FormatInt(revision, 10), "--json"})...) + next := bdShowRevision(t, bd, dir, target) + if next == revision { + t.Fatalf("a matching --if-revision on bd %s did not advance the revision", strings.Join(args, " ")) + } + revision = next + } + guarded(id, "update", id, "--priority", "3") + if got := bdShow(t, bd, dir, id); got.Priority != 3 { + t.Errorf("matching --if-revision update did not apply: priority = %d, want 3", got.Priority) + } + guarded(id, "close", id) + if got := bdShow(t, bd, dir, id); got.Status != types.StatusClosed { + t.Errorf("matching --if-revision close did not apply: status = %s, want closed", got.Status) + } + bdRunOK(t, bd, dir, "delete", id, "--force", "--if-revision", strconv.FormatInt(revision, 10), "--json") + bdShowFail(t, bd, dir, id) + + // assign gets an issue of its own: closing an issue assigned to someone + // other than the actor is refused without --force, whatever the guard. + other := bdCreateSilent(t, bd, dir, "Guarded assign", "--type", "task") + revision = bdShowRevision(t, bd, dir, other) + guarded(other, "assign", other, "worker") + if got := bdShow(t, bd, dir, other); got.Assignee != "worker" { + t.Errorf("matching --if-revision assign did not apply: assignee = %q, want worker", got.Assignee) + } +} diff --git a/cmd/bd/version_embedded_test.go b/cmd/bd/version_embedded_test.go index a2ecfea8d..436f7a5be 100644 --- a/cmd/bd/version_embedded_test.go +++ b/cmd/bd/version_embedded_test.go @@ -3,9 +3,11 @@ package main import ( + "encoding/json" "fmt" "os" "os/exec" + "regexp" "strings" "sync" "testing" @@ -72,3 +74,96 @@ func TestEmbeddedVersionConcurrent(t *testing.T) { } } } + +// versionLinePattern is the whole of `bd version`'s stdout: +// "bd version ()", with ": " or +// ": @" inside the parentheses when the binary knows them. +var versionLinePattern = regexp.MustCompile(`^bd version (\S+) \(([^):]+)(?:: (?:\S+@)?\S+)?\)\n$`) + +// TestEmbeddedVersionOutputShape pins `bd version` the way scripts and +// plugins call it, in an ordinary workspace and in a directory that is not +// one: +// - with no arguments it exits 0 and prints exactly one line on stdout, +// "bd version (...)"; +// - --json prints one object with version, build and schema_version, plus +// commit and branch when known, and no other key; its version and build +// agree with the text line, and --format json prints the same document; +// - an argument is refused with exit 1 and points at `bd versions ` +// (#6661): on stderr in text mode, as the JSON error on stdout with +// --json, and without the version line either way. +// +// TestVersionWithAnArgumentPointsAtVersions pins the refusal on RunE; this +// pins it through the real binary, where the root pre-run and workspace +// discovery run first. +func TestEmbeddedVersionOutputShape(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + workspace, _, _ := bdInit(t, bd, "--prefix", "vo") + + for name, dir := range map[string]string{"workspace": workspace, "no workspace": t.TempDir()} { + t.Run(name, func(t *testing.T) { + text, stderr, exit := bdRunStreams(t, bd, dir, "version") + if exit != 0 { + t.Fatalf("bd version exited %d\nstdout:\n%s\nstderr:\n%s", exit, text, stderr) + } + line := versionLinePattern.FindStringSubmatch(text) + if line == nil { + t.Fatalf("bd version stdout is not one %q line:\n%q", versionLinePattern, text) + } + + doc, stderr, exit := bdRunStreams(t, bd, dir, "version", "--json") + if exit != 0 { + t.Fatalf("bd version --json exited %d\nstdout:\n%s\nstderr:\n%s", exit, doc, stderr) + } + var fields map[string]any + if err := json.Unmarshal([]byte(doc), &fields); err != nil { + t.Fatalf("bd version --json is not one JSON object: %v\n%s", err, doc) + } + for key := range fields { + switch key { + case "version", "build", "schema_version", "commit", "branch": + default: + t.Errorf("bd version --json has an unexpected key %q:\n%s", key, doc) + } + } + if _, ok := fields["schema_version"].(float64); !ok { + t.Errorf("bd version --json has no numeric schema_version:\n%s", doc) + } + if fields["version"] != line[1] || fields["build"] != line[2] { + t.Errorf("bd version --json says version %v build %v, the text line says %s and %s:\n%s", fields["version"], fields["build"], line[1], line[2], doc) + } + for _, key := range []string{"commit", "branch"} { + if v, ok := fields[key]; ok { + if s, isString := v.(string); !isString || s == "" { + t.Errorf("bd version --json %s = %#v, want a non-empty string when present", key, v) + } + } + } + if alias, _, exit := bdRunStreams(t, bd, dir, "version", "--format", "json"); exit != 0 || alias != doc { + t.Errorf("bd version --format json exited %d and printed a different document from --json:\n%s\nwant:\n%s", exit, alias, doc) + } + + stdout, stderr, exit := bdRunStreams(t, bd, dir, "version", "vo-123") + if exit != 1 || stdout != "" { + t.Errorf("bd version vo-123 exited %d with stdout %q, want exit 1 and nothing on stdout", exit, stdout) + } + for _, want := range []string{"Did you mean", "bd versions vo-123"} { + if !strings.Contains(stderr, want) { + t.Errorf("bd version vo-123 stderr lacks %q:\n%s", want, stderr) + } + } + + stdout, stderr, exit = bdRunStreams(t, bd, dir, "version", "vo-123", "--json") + var refusal struct { + Error string `json:"error"` + } + if exit != 1 || json.Unmarshal([]byte(stdout), &refusal) != nil || !strings.Contains(refusal.Error, "bd versions vo-123") { + t.Errorf("bd version vo-123 --json exited %d, want exit 1 and a JSON error naming `bd versions vo-123`\nstdout:\n%s\nstderr:\n%s", exit, stdout, stderr) + } + }) + } +} diff --git a/cmd/bd/versioned_history_switch_scan_embedded_test.go b/cmd/bd/versioned_history_switch_scan_embedded_test.go index 6c4c1a286..acf34632f 100644 --- a/cmd/bd/versioned_history_switch_scan_embedded_test.go +++ b/cmd/bd/versioned_history_switch_scan_embedded_test.go @@ -434,3 +434,53 @@ func TestEmbeddedVersionedHistorySwitchRefusesAnUnrecordableGateTimeout(t *testi t.Fatalf("after applying the remedies the switch did not turn on: config get says %q", got) } } + +// TestEmbeddedConfigSetOtherKeysSkipTheVersionedHistoryScan pins that the pre-enable scan belongs +// to versioned-history.enabled alone. In a store that holds a row the scan refuses over, `bd config +// set` and `bd config set-many` of any other key -- given the value "true", which is what turns +// the switch on -- exit 0, store the value, print no refusal and leave the switch off. The switch +// itself is still refused afterwards, which proves the store held a row the scan would have named. +// +// TestEmbeddedVersionedHistorySwitchRefusesUnversionableMetadata pins that turning the switch OFF +// never scans; this pins the other half of "any other key ... passes without reading a row" +// (checkVersionedHistoryCanBeEnabled's doc comment) through the real commands. +func TestEmbeddedConfigSetOtherKeysSkipTheVersionedHistoryScan(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + // The offender below can only be written while nothing records versions. No t.Parallel: + // t.Setenv is incompatible. + t.Setenv(vhEnvVar, "") + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "vso") + + offender := bdCreateSilent(t, bd, dir, "holds an integer past 2^53", "--metadata", `{"n":9007199254740993}`) + + for _, args := range [][]string{ + {"config", "set", "custom.vso-marker", "true"}, + {"config", "set-many", "custom.vso-batch=true"}, + } { + stdout, stderr, exit := bdRunStreams(t, bd, dir, args...) + if exit != 0 { + t.Fatalf("bd %s exited %d in a store holding %s; the scan must not run for this key\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), exit, offender, stdout, stderr) + } + if out := stdout + stderr; strings.Contains(out, "cannot turn versioned history on") || strings.Contains(out, offender) { + t.Errorf("bd %s printed the versioned-history refusal for a key that is not the switch:\n%s", strings.Join(args, " "), out) + } + } + for key, want := range map[string]string{"custom.vso-marker": "true", "custom.vso-batch": "true"} { + if got := strings.TrimSpace(bdConfig(t, bd, dir, "get", key)); got != want { + t.Errorf("config get %s = %q, want %q", key, got, want) + } + } + if got := strings.TrimSpace(bdConfig(t, bd, dir, "get", "versioned-history.enabled")); strings.HasPrefix(got, "true") { + t.Fatalf("setting other keys turned versioned history on: config get says %q", got) + } + + // The control: the scan does see the offender when the switch itself is set. + out, _ := bdRunFailCode(t, bd, dir, "config", "set", "versioned-history.enabled", "true") + if !strings.Contains(out, offender) { + t.Fatalf("the refused switch does not name %s, so the pin above proves nothing:\n%s", offender, out) + } +} From 05fffb1e11b4a60a298d884a694a1bd4e8da8152 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 03:47:00 +0000 Subject: [PATCH 19/54] test(storage): versioned-history flag-off differential and statement pins (#6132, #6135) (#7681) * test(domain/db): pin the statements the history-off upsert Insert issues The participation_generation write fence (#6943) gave domain/db Insert's upsert path (CreateOnly unset: the quick, todo, gate and batch proxied servers' create) an existence probe ahead of the upsert, skipped while versioned history is off. Pin that skip with a sqlmock statement-sequence test: with history off, the upsert into issues and into wisps issues exactly the three statements the pre-stack code (4a67d0d028) issues -- the upsert, the created event's same-content scan and its insert -- and nothing else. The mock is reached through internal/storage/sqlcount, and the pin also requires exactly three statements to reach the driver. sqlmock alone answers an unscripted statement with an error and fails only if the caller returns it, so a best-effort read whose error Insert dropped would pass its checks; the count catches it. The history-on run of the same script is the control: the probe (and the mint after it) must make the pin refuse it, which shows the script is exhaustive and that Insert reads the transaction's scope. Passes on 4a67d0d028 and on main; main issues no extra read with history off. Before-parity pin P2 (Insert half) from the fork gap analysis, for gastownhall/beads#6132 / #6135. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(issueops): pin the statements the history-off defer-wake sweep issues Port the fork's sqlmock pin of WakeExpiredDefersInTx with versioned history off (donnabox/codex/preview3-source-catchup-46aa 11c74cb575): a one-row sweep issues exactly the expired-row SELECT, the UPDATE, the event's same-content SELECT and INSERT, and the empty wisps pass, so any read added before or after waking a row while history is off fails the pin. It passes on main unchanged and guards the fork's wake-skip change, which checks each expired row against the mint before waking it and must keep that check free with history off. Adapted for main: the doc no longer describes the fork's pre-wake check as existing; a history-on run of the same script is added as the control (the mint's read must make the pin refuse it), so the pin cannot pass vacuously; and the mock is reached through internal/storage/sqlcount, so the pin also requires exactly five statements to reach the driver. That catches a read whose error the sweep swallowed, which sqlmock alone does not see. Passes on 4a67d0d028 and on main. Before-parity pin P2 (wake half), for gastownhall/beads#6132 / #6135. Co-authored-by: Jim Wordelman Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(storage): versioned-history flag-off differential against pre-stack goldens gastownhall/beads#6132 and #6135 require a differential test proving that with versioned history off (the default) storage behaves byte-identically to before the feature. Existing coverage stops short of that: TestDualWriteContractFlagOff only counts version rows, the participation-generation contracts check one column, and createbatchequiv runs its batch with history on over a column subset. Add internal/storage/historyoffequiv. Each leg (embedded Dolt, Dolt server, unit of work) runs one scripted mutation set through its own front doors on a store whose history was never turned on: - on the issues plane, every write role: creates, batch create, apply batch, updates, the no-op update, reparent, dependency add/remove, comments, claim/heartbeat/release, lease reclaim, metadata CAS, close/reopen, batch close with a refused item, defer and wake, delete; - the persistence moves in both directions; - on the wisps plane, the roles again: create, update and the no-op update, dependency add/remove, comment, metadata CAS, close/reopen, apply batch, batch close, release, defer and wake (the sweep's wisps pass), delete, a forced close of a no-history wisp, and the refusals the lease roles' contracts give a wisp (Claim, Heartbeat; Reclaim finds nothing); - each leg's own surface: the store legs' DoltStorage methods (import upsert, promotion, RunInTransaction, ...) and the uow leg's CreateOnly-unset domain upsert and importer. Neither reclaim sweep finds an expired lease, since no role can age one. The harness then dumps every column of every base table, plus the scenario's Dolt commit messages and the final dolt_status, and compares per-table SHA-256 digests with goldens recorded by running the same harness against origin/main 4a67d0d028, the commit before the stack (#6943, #6661, #6664). The only tolerated differences are listed in schemaAdditions: the stack's new columns (issues/wisps.participation_generation, issue_versions.removed_restriction) must be NULL in every row and are then left out of the digest; its new table (epoch_minted_addresses) must be empty; its own migration bookkeeping rows (0070-0072, ignored/0028) may each be there once. Recording and checking apply that rule the same way, so a golden re-recorded from main is byte-identical to the pre-stack one (verified on all three legs). The package doc says how the goldens are kept: a later migration whose schema flag-off leaves untouched adds its own schemaAdditions entry; a deliberate flag-off change, a scenario change or an engine rendering change re-records them and shows the dump diff. Wall-clock instants, wall-clock-derived ids and random tokens are masked by explicit, documented rules found by diffing two runs; instants the scenario supplies stay verbatim, so a write that starts overwriting one still shows. All three legs pass on main. Before-parity pin P1 from the fork gap analysis; the new embedded test is placed in the 40-shard storage block. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Jim Wordelman --- .../scripts/embedded-storage-test-shards.txt | 1 + BUILD.bazel | 4 + internal/storage/BUILD.bazel | 1 + internal/storage/dolt/BUILD.bazel | 2 + .../dolt/history_off_differential_test.go | 35 ++ internal/storage/domain/db/BUILD.bazel | 2 + .../db/issue_insert_history_off_test.go | 143 +++++ internal/storage/embeddeddolt/BUILD.bazel | 2 + .../history_off_differential_test.go | 39 ++ internal/storage/historyoffequiv/BUILD.bazel | 96 ++++ internal/storage/historyoffequiv/dump.go | 254 +++++++++ .../historyoffequiv/golden/embedded.json | 539 ++++++++++++++++++ .../historyoffequiv/golden/server.json | 539 ++++++++++++++++++ .../storage/historyoffequiv/golden/uow.json | 531 +++++++++++++++++ .../historyoffequiv/historyoffequiv.go | 528 +++++++++++++++++ internal/storage/historyoffequiv/scenario.go | 539 ++++++++++++++++++ internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/wake_defers_test.go | 127 +++++ internal/storage/uow/BUILD.bazel | 2 + .../uow/history_off_differential_test.go | 92 +++ tools/bazel/BUILD.bazel | 1 + 21 files changed, 3479 insertions(+) create mode 100644 internal/storage/dolt/history_off_differential_test.go create mode 100644 internal/storage/domain/db/issue_insert_history_off_test.go create mode 100644 internal/storage/embeddeddolt/history_off_differential_test.go create mode 100644 internal/storage/historyoffequiv/BUILD.bazel create mode 100644 internal/storage/historyoffequiv/dump.go create mode 100644 internal/storage/historyoffequiv/golden/embedded.json create mode 100644 internal/storage/historyoffequiv/golden/server.json create mode 100644 internal/storage/historyoffequiv/golden/uow.json create mode 100644 internal/storage/historyoffequiv/historyoffequiv.go create mode 100644 internal/storage/historyoffequiv/scenario.go create mode 100644 internal/storage/issueops/wake_defers_test.go create mode 100644 internal/storage/uow/history_off_differential_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index cef22090a..c3f45aacb 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -237,6 +237,7 @@ 40 10 TestGraphCounterContract 40 10 TestIgnoredCursorHealRespectsOperatorOverride 40 10 TestRunInTransactionPostCommitStageFailureIsIndeterminate +40 10 TestVersionedHistoryOffMatchesPreStackGolden_Embedded 40 10 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient 40 11 TestABumpUnderOneStoreIDMovesTheEpochSeenThroughEveryStoreID diff --git a/BUILD.bazel b/BUILD.bazel index fed381d93..0117bc8b7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -396,6 +396,7 @@ filegroup( "//internal/storage/externaldeps:repo_doc_files", "//internal/storage/fs:repo_doc_files", "//internal/storage/git:repo_doc_files", + "//internal/storage/historyoffequiv:repo_doc_files", "//internal/storage/issueops:repo_doc_files", "//internal/storage/journalscan:repo_doc_files", "//internal/storage/kvkeys:repo_doc_files", @@ -631,6 +632,7 @@ filegroup( "//internal/storage/externaldeps:repo_go_srcs", "//internal/storage/fs:repo_go_srcs", "//internal/storage/git:repo_go_srcs", + "//internal/storage/historyoffequiv:repo_go_srcs", "//internal/storage/issueops:repo_go_srcs", "//internal/storage/journalscan:repo_go_srcs", "//internal/storage/kvkeys:repo_go_srcs", @@ -851,6 +853,7 @@ filegroup( "//internal/storage/externaldeps:repo_go_test_srcs", "//internal/storage/fs:repo_go_test_srcs", "//internal/storage/git:repo_go_test_srcs", + "//internal/storage/historyoffequiv:repo_go_test_srcs", "//internal/storage/issueops:repo_go_test_srcs", "//internal/storage/journalscan:repo_go_test_srcs", "//internal/storage/kvkeys:repo_go_test_srcs", @@ -1080,6 +1083,7 @@ filegroup( "//internal/storage/externaldeps:repo_other_files", "//internal/storage/fs:repo_other_files", "//internal/storage/git:repo_other_files", + "//internal/storage/historyoffequiv:repo_other_files", "//internal/storage/issueops:repo_other_files", "//internal/storage/journalscan:repo_other_files", "//internal/storage/kvkeys:repo_other_files", diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 73737e702..23a5a075f 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -170,6 +170,7 @@ filegroup( "//internal/storage/externaldeps:go_srcs", "//internal/storage/fs:go_srcs", "//internal/storage/git:go_srcs", + "//internal/storage/historyoffequiv:go_srcs", "//internal/storage/issueops:go_srcs", "//internal/storage/journalscan:go_srcs", "//internal/storage/kvkeys:go_srcs", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 773a53d6b..0ab26a75e 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -244,6 +244,7 @@ go_test( "gm2g3g5r_leak_repro_test.go", "graph_counter_contract_test.go", "heartbeater_contract_test.go", + "history_off_differential_test.go", "history_presence_contract_test.go", "history_test.go", "identity_mismatch_message_test.go", @@ -414,6 +415,7 @@ go_test( "//internal/storage/depid", "//internal/storage/doltutil", "//internal/storage/domain", + "//internal/storage/historyoffequiv", "//internal/storage/issueops", "//internal/storage/journalscan", "//internal/storage/rowid", diff --git a/internal/storage/dolt/history_off_differential_test.go b/internal/storage/dolt/history_off_differential_test.go new file mode 100644 index 000000000..f4bcefae9 --- /dev/null +++ b/internal/storage/dolt/history_off_differential_test.go @@ -0,0 +1,35 @@ +package dolt + +import ( + "context" + "database/sql" + "testing" + + "github.com/steveyegge/beads/internal/storage/historyoffequiv" +) + +// TestVersionedHistoryOffMatchesPreStackGolden_Dolt is the server leg's +// versioned-history flag-off differential (internal/storage/historyoffequiv): +// on a store whose versioned history was never turned on, the scripted +// mutation set must leave every table as the code before the +// versioned-history stack left it, apart from that stack's new columns and +// table staying NULL and empty. +func TestVersionedHistoryOffMatchesPreStackGolden_Dolt(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + t.Cleanup(cleanup) + if err := store.SetConfig(t.Context(), "issue_prefix", historyoffequiv.Prefix); err != nil { + t.Fatalf("set issue_prefix: %v", err) + } + historyoffequiv.Run(t, historyoffequiv.Leg{ + Name: "server", + Roles: store, + Wake: func(ctx context.Context) error { + store.WakeExpiredDefersAdvisory(ctx) + return nil + }, + Extra: historyoffequiv.DoltStorageSteps(store), + Query: historyoffequiv.SQLQuery(func(context.Context) (*sql.DB, func(), error) { + return store.DB(), func() {}, nil + }), + }) +} diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 0b7df5abb..22f93bff7 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -67,6 +67,7 @@ go_test( "issue_cross_tier_test.go", "issue_descendants_test.go", "issue_force_close_policy_test.go", + "issue_insert_history_off_test.go", "issue_mergeops_test.go", "issue_scan_parity_test.go", "issue_search_counts_missing_table_test.go", @@ -114,6 +115,7 @@ go_test( "//internal/storage/journalscan", "//internal/storage/schema", "//internal/storage/sqlbuild", + "//internal/storage/sqlcount", "//internal/testutil", "//internal/types", "//issueops", diff --git a/internal/storage/domain/db/issue_insert_history_off_test.go b/internal/storage/domain/db/issue_insert_history_off_test.go new file mode 100644 index 000000000..355cf5598 --- /dev/null +++ b/internal/storage/domain/db/issue_insert_history_off_test.go @@ -0,0 +1,143 @@ +package db + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "sync/atomic" + "testing" + "time" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" +) + +// TestInsertUpsertWithHistoryOffIssuesThePreStackStatements pins the +// statements Insert's upsert (CreateOnly unset: the quick, todo, gate and +// batch proxied servers' create) issues while versioned history is off. +// +// The participation_generation write fence (#6943) gave this path an +// existence probe (Exists) before the upsert, so the mint can tell a new row +// from one the upsert resolved to. With history off neither mint entry reads +// anything, so the probe is skipped and the create costs the round trips it +// did before the fence. Those are exactly the three statements below, the +// same three the code before the versioned-history stack (4a67d0d028) +// issues: the upsert, the created event's same-content scan, and its insert. +// The events journal is off too (the default); it adds its own statements +// whatever history does. +// +// Nothing else is scripted, and the mock is reached through sqlcount, which +// counts every statement Insert sends. So a probe, or any other read added +// ahead of the upsert or after it while history is off, fails the pin even +// when Insert swallows the error sqlmock answers it with: sqlmock's own +// checks see only an error the caller returns and expectations left unmet, +// and the count sees the rest. The history-on run of the same script is the +// control: there Insert reads (the probe, then the mint), so the pin has to +// refuse it, which shows that the script is exhaustive and that Insert reads +// the activation this transaction is scoped with. +func TestInsertUpsertWithHistoryOffIssuesThePreStackStatements(t *testing.T) { + for _, tc := range []struct { + name string + wisp bool + table, events string + }{ + {name: "issues table", table: "issues", events: "events"}, + {name: "wisps table", wisp: true, table: "wisps", events: "wisp_events"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := insertUpsertAgainstScript(t, tc.wisp, tc.table, tc.events, false); err != nil { + t.Fatalf("history off: %v", err) + } + }) + } + t.Run("history on is refused by the same script", func(t *testing.T) { + err := insertUpsertAgainstScript(t, false, "issues", "events", true) + if err == nil { + t.Fatal("Insert with history on passed the history-off script; " + + "the script no longer sees the reads history adds, so the history-off cases above prove nothing") + } + t.Logf("history on, as expected: %v", err) + }) +} + +// upsertStatements is how many statements the script below holds, and so +// how many the history-off upsert may send. +const upsertStatements = 3 + +// insertUpsertAgainstScript runs one upsert Insert into table on a mocked +// transaction scoped with versioned history as asked, against the script of +// the three statements the history-off upsert issues. It returns why the +// Insert broke the script, or nil if it sent exactly the scripted statements. +func insertUpsertAgainstScript(t *testing.T, wisp bool, table, events string, history bool) error { + t.Helper() + mock, tx, counts := countedMockTx(t) + // Every store and provider scopes each transaction with its own setting; + // a store whose switch is off scopes false. + t.Cleanup(issueops.ScopeVersionedHistoryTransaction(tx, history)) + + mock.ExpectExec(`^INSERT INTO ` + table + ` \( id, content_hash, .* ON DUPLICATE KEY UPDATE .* row_lock = VALUES\(row_lock\)$`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`^SELECT id FROM `+events+` WHERE issue_id = \? AND event_type = \? AND actor = \?`). + WithArgs("bd-1", string(types.EventCreated), "creator", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectExec(`^INSERT INTO ` + events + ` \(id, issue_id, event_type, actor, old_value, new_value, comment, created_at\)`). + WillReturnResult(sqlmock.NewResult(0, 1)) + + at := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) + if err := NewIssueSQLRepository(tx).Insert(t.Context(), &types.Issue{ + ID: "bd-1", Title: "upserted", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + CreatedAt: at, UpdatedAt: at, + }, "creator", domain.InsertIssueOpts{UseWispsTable: wisp}); err != nil { + return fmt.Errorf("the upsert failed against the script: %w", err) + } + if n := counts.Total(); n != upsertStatements { + return fmt.Errorf("the upsert sent %d statements, want exactly the %d scripted; "+ + "sqlmock refused each extra one with an error Insert dropped", n, upsertStatements) + } + if err := mock.ExpectationsWereMet(); err != nil { + return fmt.Errorf("the upsert did not send the scripted statements: %w", err) + } + return nil +} + +var sqlmockSeq atomic.Int64 + +// countedMockTx begins a transaction on a fresh sqlmock connection that is +// reached through sqlcount, so a pin can count every statement the code +// under test sends, including one whose error it swallowed. +func countedMockTx(t *testing.T) (sqlmock.Sqlmock, *sql.Tx, *sqlcount.Counts) { + t.Helper() + dsn := fmt.Sprintf("domain_db_counted_%d", sqlmockSeq.Add(1)) + mockDB, mock, err := sqlmock.NewWithDSN(dsn) + if err != nil { + t.Fatalf("sqlmock.NewWithDSN: %v", err) + } + counts := &sqlcount.Counts{} + db := sql.OpenDB(sqlcount.WrapConnector(sqlmockConnector{dsn: dsn, driver: mockDB.Driver()}, counts)) + t.Cleanup(func() { + _ = db.Close() + _ = mockDB.Close() + }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + t.Cleanup(func() { _ = tx.Rollback() }) + return mock, tx, counts +} + +// sqlmockConnector opens the sqlmock connection registered under dsn, for +// sql.OpenDB through a wrapping connector. +type sqlmockConnector struct { + dsn string + driver driver.Driver +} + +func (c sqlmockConnector) Connect(context.Context) (driver.Conn, error) { return c.driver.Open(c.dsn) } +func (c sqlmockConnector) Driver() driver.Driver { return c.driver } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 0391b56e9..5be835710 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -162,6 +162,7 @@ go_test( "graph_counter_contract_test.go", "graph_walk_plan_test.go", "heartbeater_contract_test.go", + "history_off_differential_test.go", "history_presence_contract_test.go", "history_test.go", "is_blocked_batch_test.go", @@ -267,6 +268,7 @@ go_test( "//internal/storage/doltutil", "//internal/storage/domain/db", "//internal/storage/externaldeps", + "//internal/storage/historyoffequiv", "//internal/storage/issueops", "//internal/storage/schema", "//internal/storage/sqlcount", diff --git a/internal/storage/embeddeddolt/history_off_differential_test.go b/internal/storage/embeddeddolt/history_off_differential_test.go new file mode 100644 index 000000000..505620270 --- /dev/null +++ b/internal/storage/embeddeddolt/history_off_differential_test.go @@ -0,0 +1,39 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/historyoffequiv" +) + +// TestVersionedHistoryOffMatchesPreStackGolden_Embedded is the embedded +// leg's versioned-history flag-off differential (internal/storage/historyoffequiv): +// on a store whose versioned history was never turned on, the scripted +// mutation set must leave every table as the code before the +// versioned-history stack left it, apart from that stack's new columns and +// table staying NULL and empty. +func TestVersionedHistoryOffMatchesPreStackGolden_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, historyoffequiv.Prefix) + historyoffequiv.Run(t, historyoffequiv.Leg{ + Name: "embedded", + Roles: te.store, + Wake: func(ctx context.Context) error { + te.store.WakeExpiredDefersAdvisory(ctx) + return nil + }, + Extra: historyoffequiv.DoltStorageSteps(te.store), + Query: historyoffequiv.SQLQuery(func(ctx context.Context) (*sql.DB, func(), error) { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + return nil, nil, err + } + return db, func() { _ = cleanup() }, nil + }), + }) +} diff --git a/internal/storage/historyoffequiv/BUILD.bazel b/internal/storage/historyoffequiv/BUILD.bazel new file mode 100644 index 000000000..c9e4b9a8a --- /dev/null +++ b/internal/storage/historyoffequiv/BUILD.bazel @@ -0,0 +1,96 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "historyoffequiv", + srcs = [ + "dump.go", + "historyoffequiv.go", + "scenario.go", + ], + embedsrcs = [ + "golden/embedded.json", + "golden/server.json", + "golden/uow.json", + ], + importpath = "github.com/steveyegge/beads/internal/storage/historyoffequiv", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//issueops", + ], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/internal/storage/historyoffequiv/dump.go b/internal/storage/historyoffequiv/dump.go new file mode 100644 index 000000000..b5dbb36e9 --- /dev/null +++ b/internal/storage/historyoffequiv/dump.go @@ -0,0 +1,254 @@ +package historyoffequiv + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "testing" + "time" +) + +// Every column of every base table is dumped: there is no list of tables to +// keep up to date, so a table a later change starts writing is compared the +// moment it exists. Each column is cast to text in SQL by its declared type +// (datetimes to a fixed format with microseconds, JSON to its text, binary to +// hex), and the text is then normalized only where a value is not a function +// of the scenario: +// +// - A datetime inside the run's window (the instants the scenario's writes +// stamp from the wall clock) reads "", and so does an RFC 3339 +// instant inside the window embedded in text (an event's snapshot of the +// row it changed). NULL stays NULL, and an instant the scenario supplied +// (fixedAt, decades outside the window) stays verbatim, so a write that +// starts or stops stamping a column, or that overwrites a supplied +// instant, still changes the digest. +// - A JSON value is re-serialized with sorted keys, so two engines' (or two +// versions') spacing and key order do not read as a difference. +// - The columns in volatileColumns hold values minted from randomness or +// derived from a wall-clock instant. Each reads "" under its +// rule, which keeps NULL, the empty string and the "none" value apart +// from a minted one. +// +// Rows are compared as a sorted multiset per table. + +type volatility int + +const ( + // volatileAlways masks every non-empty value. + volatileAlways volatility = iota + // volatileNonZero masks every value but "0", the column's "none". + volatileNonZero + // volatileUUID masks a UUID-shaped value and keeps any other verbatim. + volatileUUID +) + +// volatileColumns are the columns whose values differ between two runs of the +// same scenario on the same code, each with why. They were found by running +// the scenario twice on each leg and diffing the dumps. +var volatileColumns = map[string]map[string]volatility{ + // An event's id is a digest over its content and its wall-clock + // created_at (issueops.InsertDerivedEvent). + "events": {"id": volatileAlways}, + "wisp_events": {"id": volatileAlways}, + // A live comment's id is the same kind of digest; an imported comment + // keeps the id it arrived with, which stays verbatim. + "comments": {"id": volatileUUID}, + "wisp_comments": {"id": volatileUUID}, + // row_lock is a fresh random CAS token on every write; 0 means none. + "issues": {"row_lock": volatileNonZero}, + "wisps": {"row_lock": volatileNonZero}, + // Per-session coordination tokens (dependency-coordination/v1/..., + // issue-create/v1/...) are random; their keys are digests of the ids + // involved and stay verbatim. + "local_metadata": {"value": volatileAlways}, +} + +var ( + uuidRe = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + rfc3339Re = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})`) +) + +func maskVolatile(table, column string, v string) (string, bool) { + rule, ok := volatileColumns[table][column] + if !ok || v == "" { + return v, false + } + switch rule { + case volatileNonZero: + if v == "0" { + return v, false + } + case volatileUUID: + if !uuidRe.MatchString(v) { + return v, false + } + } + return "", true +} + +// Two pseudo-tables carry the Dolt commit graph, compared like any table: the +// messages of the commits the scenario made, in order (a change to what a +// write commits, or whether it commits at all, shows up there), and the +// working set it left behind (a change to what a write stages shows up +// there). +const ( + doltLogTable = "@dolt_log" + doltStatusTable = "@dolt_status" +) + +func doltLogCount(t *testing.T, ctx context.Context, q Query) int { + t.Helper() + rows, err := q(ctx, "SELECT CAST(COUNT(*) AS CHAR) FROM dolt_log") + if err != nil { + t.Fatalf("count dolt_log: %v", err) + } + n, err := strconv.Atoi(*rows[0][0]) + if err != nil { + t.Fatalf("count dolt_log: %v", err) + } + return n +} + +func dumpAll(t *testing.T, ctx context.Context, q Query, windowStart, windowEnd time.Time, logBefore int) dump { + t.Helper() + tableRows, err := q(ctx, `SELECT table_name FROM information_schema.tables + WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE' ORDER BY table_name`) + if err != nil { + t.Fatalf("list tables: %v", err) + } + colRows, err := q(ctx, `SELECT table_name, column_name, data_type FROM information_schema.columns + WHERE table_schema = DATABASE() ORDER BY table_name, ordinal_position`) + if err != nil { + t.Fatalf("list columns: %v", err) + } + type column struct{ name, dataType string } + columns := map[string][]column{} + for _, r := range colRows { + columns[*r[0]] = append(columns[*r[0]], column{name: *r[1], dataType: strings.ToLower(*r[2])}) + } + out := dump{Tables: map[string]tableDump{}} + for _, r := range tableRows { + table := *r[0] + if strings.HasPrefix(table, "dolt_") { + continue // Dolt's own system tables; the commit graph is read below. + } + cols := columns[table] + if len(cols) == 0 { + t.Fatalf("table %s has no columns in information_schema.columns", table) + } + exprs := make([]string, len(cols)) + names := make([]string, len(cols)) + for i, c := range cols { + names[i] = c.name + exprs[i] = textExpr(c.name, c.dataType) + } + rows, err := q(ctx, fmt.Sprintf("SELECT %s FROM `%s`", strings.Join(exprs, ", "), table)) + if err != nil { + t.Fatalf("dump %s: %v", table, err) + } + for _, row := range rows { + for i, c := range cols { + row[i] = normalize(t, table, c.name, c.dataType, row[i], windowStart, windowEnd) + } + } + sortRows(rows) + out.Tables[table] = tableDump{Columns: names, Rows: rows} + } + + // The scenario's commits, newest first as dolt_log lists them, numbered so + // the sorted comparison keeps their order. The committer identity comes + // from the environment (CI sets its own), so only the message is kept. + logRows, err := q(ctx, "SELECT message FROM dolt_log") + if err != nil { + t.Fatalf("read dolt_log: %v", err) + } + made := len(logRows) - logBefore + if made < 0 { + t.Fatalf("dolt_log shrank from %d to %d commits during the scenario", logBefore, len(logRows)) + } + var log [][]*string + for i, r := range logRows[:made] { + seq := fmt.Sprintf("%04d", made-i) + log = append(log, []*string{&seq, r[0]}) + } + sortRows(log) + out.Tables[doltLogTable] = tableDump{Columns: []string{"seq", "message"}, Rows: log} + + status, err := q(ctx, "SELECT table_name, CAST(staged AS CHAR), status FROM dolt_status") + if err != nil { + t.Fatalf("read dolt_status: %v", err) + } + sortRows(status) + out.Tables[doltStatusTable] = tableDump{Columns: []string{"table_name", "staged", "status"}, Rows: status} + return out +} + +func textExpr(name, dataType string) string { + c := "`" + name + "`" + switch dataType { + case "datetime", "timestamp": + return fmt.Sprintf("DATE_FORMAT(%s, '%%Y-%%m-%%d %%H:%%i:%%s.%%f')", c) + case "date": + return fmt.Sprintf("DATE_FORMAT(%s, '%%Y-%%m-%%d')", c) + case "blob", "tinyblob", "mediumblob", "longblob", "binary", "varbinary": + return fmt.Sprintf("HEX(%s)", c) + default: + return fmt.Sprintf("CAST(%s AS CHAR)", c) + } +} + +const datetimeLayout = "2006-01-02 15:04:05.000000" + +func normalize(t *testing.T, table, column, dataType string, v *string, windowStart, windowEnd time.Time) *string { + t.Helper() + if v == nil { + return nil + } + if masked, ok := maskVolatile(table, column, *v); ok { + return &masked + } + switch dataType { + case "datetime", "timestamp": + at, err := time.Parse(datetimeLayout, *v) + if err != nil { + t.Fatalf("%s.%s: datetime %q: %v", table, column, *v, err) + } + if !at.Before(windowStart) && !at.After(windowEnd) { + s := "" + return &s + } + case "json": + var parsed any + dec := json.NewDecoder(strings.NewReader(*v)) + dec.UseNumber() // keep integers past 2^53 exact + if err := dec.Decode(&parsed); err != nil { + return v + } + b, err := json.Marshal(parsed) + if err != nil { + t.Fatalf("%s.%s: re-marshal JSON: %v", table, column, err) + } + s := string(b) + return maskInstants(&s, windowStart, windowEnd) + } + return maskInstants(v, windowStart, windowEnd) +} + +// maskInstants replaces each RFC 3339 instant inside the run's window that v +// embeds with "". +func maskInstants(v *string, windowStart, windowEnd time.Time) *string { + if !rfc3339Re.MatchString(*v) { + return v + } + s := rfc3339Re.ReplaceAllStringFunc(*v, func(m string) string { + at, err := time.Parse(time.RFC3339Nano, m) + if err != nil || at.Before(windowStart) || at.After(windowEnd) { + return m + } + return "" + }) + return &s +} diff --git a/internal/storage/historyoffequiv/golden/embedded.json b/internal/storage/historyoffequiv/golden/embedded.json new file mode 100644 index 000000000..765849833 --- /dev/null +++ b/internal/storage/historyoffequiv/golden/embedded.json @@ -0,0 +1,539 @@ +{ + "recorded_from": "4a67d0d028", + "steps": [ + "create a", + "create b blocked by a", + "create epic", + "create epic.1 under epic", + "create wisp w1", + "create no-history nh", + "create c", + "create d", + "create e", + "update a fields labels metadata", + "update a no-op", + "update w1 to persistent", + "update c to ephemeral", + "update d parent epic", + "add dependencies", + "remove dependency b-\u003ea", + "comment on a and nh", + "claim heartbeat release b", + "claim e and reclaim no lease", + "metadata compare-and-set on a", + "close reopen close epic.1", + "close batch b and refused epic", + "force close blocked no-history nh", + "create batch", + "apply batch", + "defer f1 into the past and wake it", + "delete f2", + "create wisps x y z", + "update wisp x fields labels metadata", + "update wisp x no-op", + "claim and heartbeat refuse wisps", + "reclaim sweep scoped to in-progress wisp x", + "release wisp x", + "add and remove wisp dependencies", + "comment on wisp x", + "metadata compare-and-set on wisp x", + "close reopen wisp x", + "apply batch on the wisps plane", + "close batch wisps x and nh", + "defer wisp z into the past and wake it", + "delete wisp y", + "store create s1", + "store create issues s2 s3", + "store import upsert s1 and new s4", + "store update s2", + "store update s2 no-op", + "store labels", + "store dependencies", + "store comment", + "store claim s3", + "store close and reopen s2", + "store promote wisp sw", + "store transaction", + "store delete s3" + ], + "tables": { + "@dolt_log": { + "columns": [ + "seq", + "message" + ], + "rows": 28, + "sha256": "df2b8cf059f646557d21be3a0eca54f8561525f3d7e7ee714cfb65962d31012f" + }, + "@dolt_status": { + "columns": [ + "table_name", + "staged", + "status" + ], + "rows": 3, + "sha256": "ee3e710041f54216c2ae7e45bc7774cfcaa42aff3f4d96f5996fc4a7cc419f31" + }, + "bd_events_journal": { + "columns": [ + "seq", + "ts", + "op", + "issue_id", + "issue_json", + "dep_json", + "comment_json", + "actor" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_seq": { + "columns": [ + "id", + "next_seq" + ], + "rows": 1, + "sha256": "13751b39fe07c3673f53b4f2c5ef7f2a28b26c0daed72eb7332c93c1fe520618" + }, + "child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 1, + "sha256": "753b8feea7bdc0fc74468faa597a32984edc501e9873e9f4ee1df7b59d56d0a6" + }, + "comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "d5870471fe72b8e50fe898978f24a80a23e07750ab538d47353acfabfb0e9aed" + }, + "compaction_snapshots": { + "columns": [ + "id", + "issue_id", + "compaction_level", + "snapshot_json", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "config": { + "columns": [ + "key", + "value" + ], + "rows": 10, + "sha256": "671b33231eb47e4ad897051f4e8d86d3cca88685eeaa9ced537fe24c927fdea9" + }, + "custom_statuses": { + "columns": [ + "name", + "category" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "custom_types": { + "columns": [ + "name" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "columns": [ + "id", + "issue_id", + "type", + "created_at", + "created_by", + "metadata", + "thread_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external" + ], + "rows": 6, + "sha256": "90f7dcd9895000df82d3b062d71a4c8ca81e7d5c9912f60d76515cb3eeee40d6" + }, + "events": { + "columns": [ + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at", + "id" + ], + "rows": 51, + "sha256": "898f31694ce12dcd693f374634aea358b68239f6954fdcaae3b6342c1242f2bd" + }, + "federation_peers": { + "columns": [ + "name", + "remote_url", + "username", + "password_encrypted", + "sovereignty", + "last_sync", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ignored_schema_migrations": { + "columns": [ + "version", + "applied_at", + "content_hash" + ], + "rows": 27, + "sha256": "7164db2c129cbaafbd9c426d27e809d043cafc77e8aef1f68d1760905a685acb" + }, + "interactions": { + "columns": [ + "id", + "kind", + "created_at", + "actor", + "issue_id", + "model", + "prompt", + "response", + "error", + "tool_name", + "exit_code", + "parent_id", + "label", + "reason", + "extra" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_counter": { + "columns": [ + "prefix", + "last_id" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_snapshots": { + "columns": [ + "id", + "issue_id", + "snapshot_time", + "compaction_level", + "original_size", + "compressed_size", + "original_content", + "archived_events" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_versions": { + "columns": [ + "issue_id", + "revision", + "epoch", + "durable_state", + "change_actor", + "change_agent", + "change_message", + "change_at", + "removed_at", + "removed_reason", + "attribution_status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issues": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "is_blocked", + "row_lock", + "storage_class", + "current_revision" + ], + "rows": 15, + "sha256": "1f81df40f4c4d458f20cfb6535ca13468fa03fbd10276c44752977813a4b7e20" + }, + "labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 8, + "sha256": "0102293e23eec630b67542f93c5d8da6eef5ca627cff8662970e7cb211b013ef" + }, + "leases": { + "columns": [ + "issue_id", + "holder", + "granted_at", + "lease_expires_at", + "heartbeat_at", + "granted_node" + ], + "rows": 1, + "sha256": "1eca894450e8e801850ed2cbf1c746775ad50868acaaf17e033a5da1e4be10aa" + }, + "local_metadata": { + "columns": [ + "key", + "value" + ], + "rows": 31, + "sha256": "c1d1cbc41ca9968c5c7dd380609c28126c4024868dbade26575ce105b8931fb3" + }, + "metadata": { + "columns": [ + "key", + "value" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "provenance_events": { + "columns": [ + "id", + "issue_id", + "kind", + "actor", + "ref", + "ref_kind", + "payload", + "source", + "occurred_at", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repo_mtimes": { + "columns": [ + "repo_path", + "jsonl_path", + "mtime_ns", + "last_checked" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "routes": { + "columns": [ + "prefix", + "path", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "schema_migrations": { + "columns": [ + "version", + "content_hash" + ], + "rows": 69, + "sha256": "1d7a6ae809f330f8b4c3516af50ce7604c1db3cd4c1177af3e5c2f1991d74e14" + }, + "store_epoch": { + "columns": [ + "id", + "epoch", + "bumped_at", + "bumped_reason" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "3c5f1e3eac9ff32fb1158c4421909ddd49c9131edff756c4a0e184114da93350" + }, + "wisp_dependencies": { + "columns": [ + "id", + "issue_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external", + "type", + "created_at", + "created_by", + "metadata", + "thread_id" + ], + "rows": 2, + "sha256": "80f14f79545a236513ea88f0e873f3dd477f7763aaa8cbfaa8f659a517c2a909" + }, + "wisp_events": { + "columns": [ + "id", + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at" + ], + "rows": 25, + "sha256": "1ee25446153c51b39c4de60d7f03bb56172a84e72d28df2dd44ff79aa5fc3b23" + }, + "wisp_labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 4, + "sha256": "f9c2698a90ceb3bf0ee8c4034af2845b54d65cc7bdd7638965a4682ecd104c37" + }, + "wisps": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "row_lock", + "storage_class", + "current_revision", + "is_blocked" + ], + "rows": 6, + "sha256": "f28b157579dedd4373778dc6050b2d1995e1fe30e545e5c70c6b6ccf08ed75bf" + } + } +} diff --git a/internal/storage/historyoffequiv/golden/server.json b/internal/storage/historyoffequiv/golden/server.json new file mode 100644 index 000000000..73c11eed2 --- /dev/null +++ b/internal/storage/historyoffequiv/golden/server.json @@ -0,0 +1,539 @@ +{ + "recorded_from": "4a67d0d028", + "steps": [ + "create a", + "create b blocked by a", + "create epic", + "create epic.1 under epic", + "create wisp w1", + "create no-history nh", + "create c", + "create d", + "create e", + "update a fields labels metadata", + "update a no-op", + "update w1 to persistent", + "update c to ephemeral", + "update d parent epic", + "add dependencies", + "remove dependency b-\u003ea", + "comment on a and nh", + "claim heartbeat release b", + "claim e and reclaim no lease", + "metadata compare-and-set on a", + "close reopen close epic.1", + "close batch b and refused epic", + "force close blocked no-history nh", + "create batch", + "apply batch", + "defer f1 into the past and wake it", + "delete f2", + "create wisps x y z", + "update wisp x fields labels metadata", + "update wisp x no-op", + "claim and heartbeat refuse wisps", + "reclaim sweep scoped to in-progress wisp x", + "release wisp x", + "add and remove wisp dependencies", + "comment on wisp x", + "metadata compare-and-set on wisp x", + "close reopen wisp x", + "apply batch on the wisps plane", + "close batch wisps x and nh", + "defer wisp z into the past and wake it", + "delete wisp y", + "store create s1", + "store create issues s2 s3", + "store import upsert s1 and new s4", + "store update s2", + "store update s2 no-op", + "store labels", + "store dependencies", + "store comment", + "store claim s3", + "store close and reopen s2", + "store promote wisp sw", + "store transaction", + "store delete s3" + ], + "tables": { + "@dolt_log": { + "columns": [ + "seq", + "message" + ], + "rows": 42, + "sha256": "c43f56fef8a71aa0775d5c79d962b3bc1565305b2c18c3a670941891e376811f" + }, + "@dolt_status": { + "columns": [ + "table_name", + "staged", + "status" + ], + "rows": 1, + "sha256": "8658f682f824193bcb00dd2c9744be1e86b90a249374d368903303dfb62ac195" + }, + "bd_events_journal": { + "columns": [ + "seq", + "ts", + "op", + "issue_id", + "issue_json", + "dep_json", + "comment_json", + "actor" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_seq": { + "columns": [ + "id", + "next_seq" + ], + "rows": 1, + "sha256": "13751b39fe07c3673f53b4f2c5ef7f2a28b26c0daed72eb7332c93c1fe520618" + }, + "child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 1, + "sha256": "753b8feea7bdc0fc74468faa597a32984edc501e9873e9f4ee1df7b59d56d0a6" + }, + "comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "d5870471fe72b8e50fe898978f24a80a23e07750ab538d47353acfabfb0e9aed" + }, + "compaction_snapshots": { + "columns": [ + "id", + "issue_id", + "compaction_level", + "snapshot_json", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "config": { + "columns": [ + "key", + "value" + ], + "rows": 10, + "sha256": "671b33231eb47e4ad897051f4e8d86d3cca88685eeaa9ced537fe24c927fdea9" + }, + "custom_statuses": { + "columns": [ + "name", + "category" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "custom_types": { + "columns": [ + "name" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "columns": [ + "id", + "issue_id", + "type", + "created_at", + "created_by", + "metadata", + "thread_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external" + ], + "rows": 6, + "sha256": "90f7dcd9895000df82d3b062d71a4c8ca81e7d5c9912f60d76515cb3eeee40d6" + }, + "events": { + "columns": [ + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at", + "id" + ], + "rows": 51, + "sha256": "898f31694ce12dcd693f374634aea358b68239f6954fdcaae3b6342c1242f2bd" + }, + "federation_peers": { + "columns": [ + "name", + "remote_url", + "username", + "password_encrypted", + "sovereignty", + "last_sync", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ignored_schema_migrations": { + "columns": [ + "version", + "applied_at", + "content_hash" + ], + "rows": 27, + "sha256": "7164db2c129cbaafbd9c426d27e809d043cafc77e8aef1f68d1760905a685acb" + }, + "interactions": { + "columns": [ + "id", + "kind", + "created_at", + "actor", + "issue_id", + "model", + "prompt", + "response", + "error", + "tool_name", + "exit_code", + "parent_id", + "label", + "reason", + "extra" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_counter": { + "columns": [ + "prefix", + "last_id" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_snapshots": { + "columns": [ + "id", + "issue_id", + "snapshot_time", + "compaction_level", + "original_size", + "compressed_size", + "original_content", + "archived_events" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_versions": { + "columns": [ + "issue_id", + "revision", + "epoch", + "durable_state", + "change_actor", + "change_agent", + "change_message", + "change_at", + "removed_at", + "removed_reason", + "attribution_status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issues": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "is_blocked", + "row_lock", + "storage_class", + "current_revision" + ], + "rows": 15, + "sha256": "1f81df40f4c4d458f20cfb6535ca13468fa03fbd10276c44752977813a4b7e20" + }, + "labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 8, + "sha256": "0102293e23eec630b67542f93c5d8da6eef5ca627cff8662970e7cb211b013ef" + }, + "leases": { + "columns": [ + "issue_id", + "holder", + "granted_at", + "lease_expires_at", + "heartbeat_at", + "granted_node" + ], + "rows": 1, + "sha256": "1eca894450e8e801850ed2cbf1c746775ad50868acaaf17e033a5da1e4be10aa" + }, + "local_metadata": { + "columns": [ + "key", + "value" + ], + "rows": 31, + "sha256": "c1d1cbc41ca9968c5c7dd380609c28126c4024868dbade26575ce105b8931fb3" + }, + "metadata": { + "columns": [ + "key", + "value" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "provenance_events": { + "columns": [ + "id", + "issue_id", + "kind", + "actor", + "ref", + "ref_kind", + "payload", + "source", + "occurred_at", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repo_mtimes": { + "columns": [ + "repo_path", + "jsonl_path", + "mtime_ns", + "last_checked" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "routes": { + "columns": [ + "prefix", + "path", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "schema_migrations": { + "columns": [ + "version", + "content_hash" + ], + "rows": 69, + "sha256": "1d7a6ae809f330f8b4c3516af50ce7604c1db3cd4c1177af3e5c2f1991d74e14" + }, + "store_epoch": { + "columns": [ + "id", + "epoch", + "bumped_at", + "bumped_reason" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "3c5f1e3eac9ff32fb1158c4421909ddd49c9131edff756c4a0e184114da93350" + }, + "wisp_dependencies": { + "columns": [ + "id", + "issue_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external", + "type", + "created_at", + "created_by", + "metadata", + "thread_id" + ], + "rows": 2, + "sha256": "80f14f79545a236513ea88f0e873f3dd477f7763aaa8cbfaa8f659a517c2a909" + }, + "wisp_events": { + "columns": [ + "id", + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at" + ], + "rows": 25, + "sha256": "1ee25446153c51b39c4de60d7f03bb56172a84e72d28df2dd44ff79aa5fc3b23" + }, + "wisp_labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 4, + "sha256": "f9c2698a90ceb3bf0ee8c4034af2845b54d65cc7bdd7638965a4682ecd104c37" + }, + "wisps": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "row_lock", + "storage_class", + "current_revision", + "is_blocked" + ], + "rows": 6, + "sha256": "f28b157579dedd4373778dc6050b2d1995e1fe30e545e5c70c6b6ccf08ed75bf" + } + } +} diff --git a/internal/storage/historyoffequiv/golden/uow.json b/internal/storage/historyoffequiv/golden/uow.json new file mode 100644 index 000000000..91b2d1f84 --- /dev/null +++ b/internal/storage/historyoffequiv/golden/uow.json @@ -0,0 +1,531 @@ +{ + "recorded_from": "4a67d0d028", + "steps": [ + "create a", + "create b blocked by a", + "create epic", + "create epic.1 under epic", + "create wisp w1", + "create no-history nh", + "create c", + "create d", + "create e", + "update a fields labels metadata", + "update a no-op", + "update w1 to persistent", + "update c to ephemeral", + "update d parent epic", + "add dependencies", + "remove dependency b-\u003ea", + "comment on a and nh", + "claim heartbeat release b", + "claim e and reclaim no lease", + "metadata compare-and-set on a", + "close reopen close epic.1", + "close batch b and refused epic", + "force close blocked no-history nh", + "create batch", + "apply batch", + "defer f1 into the past and wake it", + "delete f2", + "create wisps x y z", + "update wisp x fields labels metadata", + "update wisp x no-op", + "claim and heartbeat refuse wisps", + "reclaim sweep scoped to in-progress wisp x", + "release wisp x", + "add and remove wisp dependencies", + "comment on wisp x", + "metadata compare-and-set on wisp x", + "close reopen wisp x", + "apply batch on the wisps plane", + "close batch wisps x and nh", + "defer wisp z into the past and wake it", + "delete wisp y", + "domain upsert new u1", + "domain upsert existing u1", + "domain upsert new wisp uw", + "domain upsert existing wisp uw", + "import upsert u1 and new u2" + ], + "tables": { + "@dolt_log": { + "columns": [ + "seq", + "message" + ], + "rows": 30, + "sha256": "5f4e90621c1c115e95223419c9c8c8533a342469e359beeac2ec5a4b4ff7ab1b" + }, + "@dolt_status": { + "columns": [ + "table_name", + "staged", + "status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_journal": { + "columns": [ + "seq", + "ts", + "op", + "issue_id", + "issue_json", + "dep_json", + "comment_json", + "actor" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_seq": { + "columns": [ + "id", + "next_seq" + ], + "rows": 1, + "sha256": "13751b39fe07c3673f53b4f2c5ef7f2a28b26c0daed72eb7332c93c1fe520618" + }, + "child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "5b88b60c5a975432baff9ecfc6d0230e3bdc12275471a2db90c6f36b80ab4561" + }, + "compaction_snapshots": { + "columns": [ + "id", + "issue_id", + "compaction_level", + "snapshot_json", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "config": { + "columns": [ + "key", + "value" + ], + "rows": 10, + "sha256": "671b33231eb47e4ad897051f4e8d86d3cca88685eeaa9ced537fe24c927fdea9" + }, + "custom_statuses": { + "columns": [ + "name", + "category" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "custom_types": { + "columns": [ + "name" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "columns": [ + "id", + "issue_id", + "type", + "created_at", + "created_by", + "metadata", + "thread_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external" + ], + "rows": 6, + "sha256": "10891c2f5540db5ffec57100e87d971863a67023d963cda658c43c56ce368a69" + }, + "events": { + "columns": [ + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at", + "id" + ], + "rows": 39, + "sha256": "372c6143415ba862a0d951695e2c942670fc0a24a3b828d6755df97dbb34ade1" + }, + "federation_peers": { + "columns": [ + "name", + "remote_url", + "username", + "password_encrypted", + "sovereignty", + "last_sync", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ignored_schema_migrations": { + "columns": [ + "version", + "applied_at", + "content_hash" + ], + "rows": 27, + "sha256": "7164db2c129cbaafbd9c426d27e809d043cafc77e8aef1f68d1760905a685acb" + }, + "interactions": { + "columns": [ + "id", + "kind", + "created_at", + "actor", + "issue_id", + "model", + "prompt", + "response", + "error", + "tool_name", + "exit_code", + "parent_id", + "label", + "reason", + "extra" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_counter": { + "columns": [ + "prefix", + "last_id" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_snapshots": { + "columns": [ + "id", + "issue_id", + "snapshot_time", + "compaction_level", + "original_size", + "compressed_size", + "original_content", + "archived_events" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_versions": { + "columns": [ + "issue_id", + "revision", + "epoch", + "durable_state", + "change_actor", + "change_agent", + "change_message", + "change_at", + "removed_at", + "removed_reason", + "attribution_status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issues": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "is_blocked", + "row_lock", + "storage_class", + "current_revision" + ], + "rows": 12, + "sha256": "92e9eb8264175b0a6723941b3e946cd796afe4539cf9927798ce62800cab67ef" + }, + "labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 6, + "sha256": "d8cb82847c5ed046c6f264c12a51b1cc128f314fdc501aad3be0bd38cd3ec341" + }, + "leases": { + "columns": [ + "issue_id", + "holder", + "granted_at", + "lease_expires_at", + "heartbeat_at", + "granted_node" + ], + "rows": 1, + "sha256": "1eca894450e8e801850ed2cbf1c746775ad50868acaaf17e033a5da1e4be10aa" + }, + "local_metadata": { + "columns": [ + "key", + "value" + ], + "rows": 31, + "sha256": "c1d1cbc41ca9968c5c7dd380609c28126c4024868dbade26575ce105b8931fb3" + }, + "metadata": { + "columns": [ + "key", + "value" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "provenance_events": { + "columns": [ + "id", + "issue_id", + "kind", + "actor", + "ref", + "ref_kind", + "payload", + "source", + "occurred_at", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repo_mtimes": { + "columns": [ + "repo_path", + "jsonl_path", + "mtime_ns", + "last_checked" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "routes": { + "columns": [ + "prefix", + "path", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "schema_migrations": { + "columns": [ + "version", + "content_hash" + ], + "rows": 69, + "sha256": "1d7a6ae809f330f8b4c3516af50ce7604c1db3cd4c1177af3e5c2f1991d74e14" + }, + "store_epoch": { + "columns": [ + "id", + "epoch", + "bumped_at", + "bumped_reason" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "3c5f1e3eac9ff32fb1158c4421909ddd49c9131edff756c4a0e184114da93350" + }, + "wisp_dependencies": { + "columns": [ + "id", + "issue_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external", + "type", + "created_at", + "created_by", + "metadata", + "thread_id" + ], + "rows": 2, + "sha256": "80f14f79545a236513ea88f0e873f3dd477f7763aaa8cbfaa8f659a517c2a909" + }, + "wisp_events": { + "columns": [ + "id", + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at" + ], + "rows": 27, + "sha256": "34d262640e93ffdc948862667d4bc5f01a9f655bc22d8e11abdfc777731a5771" + }, + "wisp_labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 4, + "sha256": "f9c2698a90ceb3bf0ee8c4034af2845b54d65cc7bdd7638965a4682ecd104c37" + }, + "wisps": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "row_lock", + "storage_class", + "current_revision", + "is_blocked" + ], + "rows": 7, + "sha256": "f6885bac56aee7d91b48fdf56c657128ff0b530fa186fabfb52bbf271a3bc09b" + } + } +} diff --git a/internal/storage/historyoffequiv/historyoffequiv.go b/internal/storage/historyoffequiv/historyoffequiv.go new file mode 100644 index 000000000..afcdfc7f2 --- /dev/null +++ b/internal/storage/historyoffequiv/historyoffequiv.go @@ -0,0 +1,528 @@ +// Package historyoffequiv is the versioned-history flag-off differential +// (gastownhall/beads#6132, #6135: "flag-off must remain byte-identical to +// today, proven by differential test"). +// +// Each storage leg (embedded Dolt, Dolt server, unit of work) runs one +// scripted mutation set through its own front doors on a store whose +// versioned history was never turned on, the default. Run then dumps every +// column of every table, digests each table, and compares the digests with +// the leg's golden. The goldens were RECORDED FROM THE CODE BEFORE THE +// VERSIONED-HISTORY STACK (origin/main 4a67d0d028, the parent of #6943). The +// schema added since, listed in schemaAdditions under the rule spelled out +// there, is the only difference the comparison tolerates; any other +// difference in any table is a flag-off behavior change and fails the test. +// +// # Maintaining the goldens +// +// Run records instead of checking when GoldenDirEnv is set. Recording and +// checking prepare a run the same way: each schemaAdditions entry the +// database has is checked under the rule and set aside before any table is +// digested. A golden recorded from any tree whose flag-off storage behavior +// matches the committed goldens is therefore byte-identical to them apart +// from recorded_from, and recording from the current tree is also how a +// failure is read: diff its full dump against one recorded from the base. +// +// - A migration whose schema flag-off leaves untouched (nullable columns +// that stay NULL, tables that stay empty) appends an entry to +// schemaAdditions in its own PR: those columns and tables, and its +// bookkeeping rows. The goldens do not change. +// - A deliberate change to what flag-off storage writes, including a +// migration that writes rows or values while history is off, +// re-records the affected legs from its PR's tree. The PR shows the +// golden diff, and that the full dumps differ from a recording of its +// base only where the change means them to. +// - A change to the scenario re-records every leg. Record it from +// 4a67d0d028 while the harness still builds there (copy this package +// and the three legs' test files into a worktree of that commit), so +// the new steps are anchored to the pre-stack code too; once it no +// longer builds there, from the PR's base. +// - A Dolt upgrade that changes only how the engine renders a stored +// value (a JSON or datetime cast) re-records the legs that run the new +// engine from the PR's tree, and the PR shows that the full dumps +// differ from a recording of its base, made with the old engine, only +// in that rendering. The embedded leg runs the engine go.mod pins; the +// server and unit-of-work legs run the dolt binary the test finds (CI: +// DOLT_VERSION in tools/bazel/dolt.bzl). +// +// recorded_from names the tree a golden was recorded from: a commit, or a +// commit and the PR on top of it. +package historyoffequiv + +import ( + "context" + "crypto/sha256" + "database/sql" + "embed" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +// Prefix is the issue prefix every scenario id carries. A leg configures its +// store with it before Run. +const Prefix = "ho" + +// Query runs one read-only statement against the leg's database (the one its +// store writes) and returns every row, each value as text and nil for SQL +// NULL. Run casts every column to text in the statement itself, so a leg's +// driver never decides how a value is formatted. +type Query func(ctx context.Context, query string) ([][]*string, error) + +// Step is one mutation of the scenario, named for failure messages and for +// the golden's step list. +type Step struct { + Name string + Run func(ctx context.Context) error +} + +// Leg is what a backend hands Run. Its store is constructed the way the leg's +// tests construct one and is never handed SetVersionedHistoryEnabled, so +// "history off" is the default a store starts with rather than a switch +// turned off (FR-7). +type Leg struct { + // Name selects the golden: golden/.json. + Name string + // Roles is the leg's role-accessor surface. Every leg drives the same + // SharedSteps through it. + Roles Roles + // Wake runs the leg's lazy defer-wake sweep, the one write no role + // exposes. + Wake func(ctx context.Context) error + // Extra are the leg's own steps, run after the shared ones: the store + // legs' storage.DoltStorage methods (DoltStorageSteps), the unit-of-work + // leg's domain upsert. + Extra []Step + // Query reads the leg's database. + Query Query +} + +// GoldenDirEnv switches Run from checking to recording: set to a directory, +// Run writes the leg's golden there instead of comparing, with a full dump +// of the rows it digested beside it (.full.json.txt) for diffing two +// recordings by hand. RecordedFromEnv names the tree being recorded from; +// Run copies it into the golden. When to re-record is under "Maintaining the +// goldens" in the package doc. +const ( + GoldenDirEnv = "HISTORYOFFEQUIV_GOLDEN_DIR" + RecordedFromEnv = "HISTORYOFFEQUIV_RECORDED_FROM" +) + +// schemaAddition is schema added since the pre-stack code the goldens are +// anchored to that flag-off storage leaves untouched, grouped by the change +// that added it. +type schemaAddition struct { + // By names the change, for failure messages. + By string + // Columns are its nullable columns, by table. + Columns map[string][]string + // Tables are its tables. + Tables []string + // Migrations are the migration runner's bookkeeping rows for its own + // migrations, by bookkeeping table (each keyed by its version column). + Migrations map[string][]string +} + +// schemaAdditions is THE ONLY DIFFERENCE from the goldens this test +// tolerates. The rule, applied to every run before any table is digested, +// whether the run records a golden or checks one: +// +// 1. A listed column, where the database has it, must be NULL in every +// row; it is then left out of its table's dump, so the rest of every row +// still has to match the golden byte for byte. A listed column holding +// any value fails: with versioned history off nothing may write it. +// 2. A listed table, where the database has it, must be empty; it is then +// left out. +// 3. A listed bookkeeping row, where the database has it, must be there +// once; it is then left out. Every other row of a bookkeeping table is +// compared like any table's. +// +// After that, checking demands the golden's tables and columns exactly, and +// every table's digest over the golden's columns must equal the golden's. +// Nothing else is excused: not a value, not a row, not a row count. A later +// migration whose schema flag-off leaves untouched appends its own entry +// here (see "Maintaining the goldens" in the package doc). +var schemaAdditions = []schemaAddition{{ + By: "the versioned-history stack (#6943, #6661, #6664)", + Columns: map[string][]string{ + // Migration 0070: the participation_generation write fence (#6943). + // A create with history off leaves it NULL, indistinguishable from + // a legacy row (design §16.2b, FR-7). + "issues": {"participation_generation"}, + // Migration 0070 mirrors it, inert, on the wisps table (FR-8). + "wisps": {"participation_generation"}, + // Migration 0071 (#6661). + "issue_versions": {"removed_restriction"}, + }, + // Migration 0072 (#6664). + Tables: []string{"epoch_minted_addresses"}, + // 0070-0072 on the main series, and ignored/0028 (the wisps half of + // 0070) on the dolt_ignored series. + Migrations: map[string][]string{ + "schema_migrations": {"70", "71", "72"}, + "ignored_schema_migrations": {"28"}, + }, +}} + +// runWindow widens the span of the run on each side to bound the wall-clock +// instants the scenario's writes stamp. It is generous so that no time zone a +// DATETIME may be read back in, and no rounding of a sub-second instant, can +// put such an instant outside it. The scenario's own timestamps are decades +// away (fixedAt), so a column whose value moved from a scenario-supplied +// instant to "now" (or back) still shows up in the digest. +const runWindow = 36 * time.Hour + +// Run applies the shared scenario and the leg's extra steps, then checks +// every table against the leg's golden (or records it). +func Run(t *testing.T, leg Leg) { + t.Helper() + if leg.Name == "" || leg.Roles == nil || leg.Wake == nil || leg.Query == nil { + t.Fatalf("historyoffequiv.Run: leg %q needs a Name, Roles, Wake and Query", leg.Name) + } + ctx := context.Background() + before := doltLogCount(t, ctx, leg.Query) + start := time.Now().UTC() + steps := append(SharedSteps(leg.Roles, leg.Wake), leg.Extra...) + var names []string + for _, step := range steps { + if err := step.Run(ctx); err != nil { + t.Fatalf("step %q: %v", step.Name, err) + } + names = append(names, step.Name) + } + end := time.Now().UTC() + got := setAsideAdditions(t, dumpAll(t, ctx, leg.Query, start.Add(-runWindow), end.Add(runWindow), before)) + got.Steps = names + + if dir := os.Getenv(GoldenDirEnv); dir != "" { + if t.Failed() { + t.Fatalf("not recording golden/%s.json: the run breaks the schemaAdditions rule (above)", leg.Name) + } + writeGolden(t, dir, leg.Name, got) + return + } + checkGolden(t, leg.Name, got) +} + +// dump is every table's normalized rows. +type dump struct { + Steps []string + Tables map[string]tableDump +} + +type tableDump struct { + Columns []string + Rows [][]*string +} + +type golden struct { + RecordedFrom string `json:"recorded_from"` + Steps []string `json:"steps"` + Tables map[string]goldenTable `json:"tables"` +} + +type goldenTable struct { + Columns []string `json:"columns"` + Rows int `json:"rows"` + SHA256 string `json:"sha256"` +} + +//go:embed golden/*.json +var goldenFS embed.FS + +// setAsideAdditions applies the schemaAdditions rule to a run's dump: it +// checks every listed column, table and bookkeeping row the database has, +// reporting each breach, and returns the dump without them. Recording and +// checking both digest only what it returns. +func setAsideAdditions(t *testing.T, d dump) dump { + t.Helper() + columns := map[string]map[string]string{} // table -> column -> By + tables := map[string]string{} + migrations := map[string]map[string]string{} // table -> version -> By + for _, a := range schemaAdditions { + for table, cols := range a.Columns { + for _, c := range cols { + if columns[table] == nil { + columns[table] = map[string]string{} + } + columns[table][c] = a.By + } + } + for _, table := range a.Tables { + tables[table] = a.By + } + for table, versions := range a.Migrations { + for _, v := range versions { + if migrations[table] == nil { + migrations[table] = map[string]string{} + } + migrations[table][v] = a.By + } + } + } + + out := dump{Steps: d.Steps, Tables: map[string]tableDump{}} + for _, table := range sortedKeys(d.Tables) { + td := d.Tables[table] + if by, ok := tables[table]; ok { + if len(td.Rows) != 0 { + t.Errorf("table %s, added by %s, has %d rows with versioned history off, want none:\n%s", + table, by, len(td.Rows), formatRows(td.Columns, td.Rows)) + } + continue + } + var keep []int + for i, c := range td.Columns { + by, listed := columns[table][c] + if !listed { + keep = append(keep, i) + continue + } + for _, row := range td.Rows { + if row[i] != nil { + t.Errorf("%s.%s, added by %s, holds %q with versioned history off, want NULL in every row; row: %s", + table, c, by, *row[i], formatRow(row)) + } + } + } + kept := tableDump{Columns: make([]string, len(keep))} + for j, i := range keep { + kept.Columns[j] = td.Columns[i] + } + version := slices.Index(kept.Columns, "version") + if migrations[table] != nil && version < 0 { + t.Errorf("bookkeeping table %s has no version column: %v", table, td.Columns) + } + seen := map[string]int{} + for _, row := range td.Rows { + projected := make([]*string, len(keep)) + for j, i := range keep { + projected[j] = row[i] + } + if version >= 0 && projected[version] != nil { + if _, listed := migrations[table][*projected[version]]; listed { + seen[*projected[version]]++ + continue + } + } + kept.Rows = append(kept.Rows, projected) + } + for _, v := range sortedKeys(seen) { + if seen[v] != 1 { + t.Errorf("%s has %d rows for migration %s, added by %s, want at most 1", table, seen[v], v, migrations[table][v]) + } + } + sortRows(kept.Rows) + out.Tables[table] = kept + } + return out +} + +// checkGolden compares a set-aside dump with golden/.json. +func checkGolden(t *testing.T, leg string, got dump) { + t.Helper() + b, err := goldenFS.ReadFile("golden/" + leg + ".json") + if err != nil { + t.Fatalf("no golden for leg %q: record one with %s (see the package doc): %v", leg, GoldenDirEnv, err) + } + var want golden + if err := json.Unmarshal(b, &want); err != nil { + t.Fatalf("golden/%s.json: %v", leg, err) + } + from := fmt.Sprintf("golden/%s.json (recorded from %s)", leg, want.RecordedFrom) + if !slices.Equal(got.Steps, want.Steps) { + t.Fatalf("the scenario's steps differ from those %s ran, so its digests no longer describe this scenario; "+ + "re-record it (\"Maintaining the goldens\" in the package doc):\ngot: %q\nwant: %q", + from, got.Steps, want.Steps) + } + unexplained := "; a migration whose schema flag-off leaves untouched lists it in schemaAdditions, " + + "and a change to what flag-off writes re-records the goldens (\"Maintaining the goldens\" in the package doc)" + for _, table := range sortedKeys(got.Tables) { + if _, ok := want.Tables[table]; !ok { + t.Errorf("table %s is not in %s and not in schemaAdditions%s:\n%s", + table, from, unexplained, formatRows(got.Tables[table].Columns, got.Tables[table].Rows)) + } + } + for _, table := range sortedKeys(want.Tables) { + wt := want.Tables[table] + gt, ok := got.Tables[table] + if !ok { + t.Errorf("table %s is in %s but not in this database", table, from) + continue + } + index := map[string]int{} + for i, c := range gt.Columns { + index[c] = i + } + ok = true + for _, c := range wt.Columns { + if _, present := index[c]; !present { + t.Errorf("%s.%s is in %s but not in this database", table, c, from) + ok = false + } + } + for _, c := range gt.Columns { + if !slices.Contains(wt.Columns, c) { + t.Errorf("%s.%s is not in %s and not in schemaAdditions%s", table, c, from, unexplained) + ok = false + } + } + if !ok { + continue + } + projected := make([][]*string, 0, len(gt.Rows)) + for _, row := range gt.Rows { + p := make([]*string, len(wt.Columns)) + for j, c := range wt.Columns { + p[j] = row[index[c]] + } + projected = append(projected, p) + } + sortRows(projected) + if d := digest(projected); d != wt.SHA256 || len(projected) != wt.Rows { + t.Errorf("%s differs from %s: got %d rows %s, want %d rows %s; rows now (columns %v):\n%s", + table, from, len(projected), d, wt.Rows, wt.SHA256, wt.Columns, formatRows(wt.Columns, projected)) + } + } +} + +// writeGolden records a set-aside dump as golden/.json in dir. +func writeGolden(t *testing.T, dir, leg string, d dump) { + t.Helper() + g := golden{RecordedFrom: os.Getenv(RecordedFromEnv), Steps: d.Steps, Tables: map[string]goldenTable{}} + if g.RecordedFrom == "" { + t.Fatalf("%s is set but %s is not: name the tree this golden is recorded from", GoldenDirEnv, RecordedFromEnv) + } + full := map[string][]string{} + for table, td := range d.Tables { + g.Tables[table] = goldenTable{Columns: td.Columns, Rows: len(td.Rows), SHA256: digest(td.Rows)} + full[table] = append(full[table], strings.Join(td.Columns, " | ")) + for _, row := range td.Rows { + full[table] = append(full[table], formatRow(row)) + } + } + b, err := json.MarshalIndent(g, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, leg+".json"), append(b, '\n'), 0o600); err != nil { + t.Fatal(err) + } + fb, _ := json.MarshalIndent(full, "", " ") + _ = os.WriteFile(filepath.Join(dir, leg+".full.json.txt"), fb, 0o600) +} + +func digest(rows [][]*string) string { + h := sha256.New() + for _, row := range rows { + h.Write([]byte(formatRow(row))) + h.Write([]byte{'\n'}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func formatRow(row []*string) string { + parts := make([]string, len(row)) + for i, v := range row { + if v == nil { + parts[i] = "NULL" + } else { + parts[i] = strconv.Quote(*v) + } + } + return strings.Join(parts, " ") +} + +func formatRows(columns []string, rows [][]*string) string { + lines := []string{" " + strings.Join(columns, " | ")} + for _, row := range rows { + lines = append(lines, " "+formatRow(row)) + } + return strings.Join(lines, "\n") +} + +func sortRows(rows [][]*string) { + sort.SliceStable(rows, func(i, j int) bool { return formatRow(rows[i]) < formatRow(rows[j]) }) +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// SQLQuery adapts a leg that reads through database/sql. open returns a +// handle on the leg's database and a function that releases it. +func SQLQuery(open func(ctx context.Context) (*sql.DB, func(), error)) Query { + return func(ctx context.Context, query string) ([][]*string, error) { + db, release, err := open(ctx) + if err != nil { + return nil, err + } + defer release() + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + var out [][]*string + for rows.Next() { + raw := make([]sql.NullString, len(cols)) + dest := make([]any, len(cols)) + for i := range raw { + dest[i] = &raw[i] + } + if err := rows.Scan(dest...); err != nil { + return nil, err + } + row := make([]*string, len(cols)) + for i, v := range raw { + if v.Valid { + s := v.String + row[i] = &s + } + } + out = append(out, row) + } + return out, rows.Err() + } +} + +// AnyRows converts rows of driver values (a raw-SQL use case's result) to +// Query's shape. +func AnyRows(rows [][]any) [][]*string { + out := make([][]*string, 0, len(rows)) + for _, r := range rows { + row := make([]*string, len(r)) + for i, v := range r { + switch v := v.(type) { + case nil: + case []byte: + s := string(v) + row[i] = &s + case string: + s := v + row[i] = &s + default: + s := fmt.Sprint(v) + row[i] = &s + } + } + out = append(out, row) + } + return out +} diff --git a/internal/storage/historyoffequiv/scenario.go b/internal/storage/historyoffequiv/scenario.go new file mode 100644 index 000000000..90dadd04b --- /dev/null +++ b/internal/storage/historyoffequiv/scenario.go @@ -0,0 +1,539 @@ +package historyoffequiv + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// Roles is the role-accessor surface all three legs offer. The shared +// scenario reaches storage only through it (and Leg.Wake), so each leg runs +// the scenario through its own implementations: the embedded store's and the +// server store's issueops-backed roles, and the unit-of-work provider's +// domain-backed ones. +type Roles interface { + IssueLifecycle() (publicops.Lifecycle, error) + BatchCreator() (publicops.BatchCreator, error) + BatchApplier() (publicops.BatchApplier, error) + BatchCloser() (publicops.BatchCloser, error) + DependencyEditor() (publicops.DependencyEditor, error) + Commenter() (publicops.Commenter, error) + IssueClaimer() (publicops.Claimer, error) + Heartbeater() (publicops.Heartbeater, error) + Releaser() (publicops.Releaser, error) + LeaseReclaimer() (publicops.LeaseReclaimer, error) + MetadataCAS() (publicops.MetadataCAS, error) + Deleter() (publicops.Deleter, error) +} + +// fixedAt is every timestamp the scenario supplies. It is decades outside +// any run's window, so a column that keeps it reads it verbatim in the dump +// and a column that loses it to the wall clock reads "". +var fixedAt = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) + +func id(suffix string) string { return Prefix + "-" + suffix } + +func newIssue(suffix, title string, issueType types.IssueType) *types.Issue { + return &types.Issue{ + ID: id(suffix), Title: title, Description: "about " + suffix, Status: types.StatusOpen, + Priority: 2, IssueType: issueType, CreatedAt: fixedAt, UpdatedAt: fixedAt, + } +} + +func set[T any](v T) publicops.Field[T] { return publicops.Field[T]{Set: true, Value: v} } + +func raw(s string) *json.RawMessage { + m := json.RawMessage(s) + return &m +} + +// SharedSteps is the scenario every leg runs: each write role on the issues +// plane, then each again on the wisps plane (an ephemeral wisp, and the +// no-history wisp nh along the way), including the paths the +// versioned-history stack rewired (single and batch creates, the persistence +// moves between planes, the defer wake), refusals that must write nothing (a +// batch close item close policy refuses), and no-op updates that must leave +// every row alone. On the wisps plane the lease roles refuse (Claim, +// Heartbeat) or find nothing (Reclaim), as their contracts say, and those +// steps check the refusal. Neither reclaim sweep finds an expired lease: a +// role cannot age one, so they cover the sweep's reads and commit, not a +// revert. +func SharedSteps(r Roles, wake func(context.Context) error) []Step { + create := func(req publicops.CreateRequest) func(context.Context) error { + req.Actor = "alice" + return func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + _, err = lifecycle.Create(ctx, req) + return err + } + } + update := func(issueID string, patch publicops.IssuePatch) func(context.Context) error { + return func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + _, err = lifecycle.Update(ctx, publicops.UpdateRequest{Actor: "bob", IssueID: issueID, Patch: patch}) + return err + } + } + + a := newIssue("a", "alpha", types.TypeTask) + a.Labels = []string{"one", "two"} + a.Metadata = json.RawMessage(`{"k":1,"s":"x"}`) + a.Notes = "first note" + b := newIssue("b", "bravo", types.TypeBug) + b.Priority = 1 + epic := newIssue("epic", "the epic", types.TypeEpic) + child := newIssue("epic.1", "epic child", types.TypeTask) + w1 := newIssue("w1", "a wisp", types.TypeTask) + w1.Ephemeral = true + w1.Labels = []string{"wl"} + nh := newIssue("nh", "no history", types.TypeTask) + nh.NoHistory = true + c := newIssue("c", "charlie", types.TypeFeature) + c.Labels = []string{"moving"} + d := newIssue("d", "delta", types.TypeTask) + e := newIssue("e", "echo", types.TypeTask) + // The wisps-plane block's wisps: x carries the field, lease, comment, + // metadata and close traffic, y the refused claim and the dependency, + // apply and delete traffic, z the defer wake. + wisp := func(suffix, title string) *types.Issue { + w := newIssue(suffix, title, types.TypeTask) + w.Ephemeral = true + return w + } + x := wisp("x", "x-ray") + x.Labels = []string{"xl", "xm"} + x.Metadata = json.RawMessage(`{"k":1}`) + y, z := wisp("y", "yankee"), wisp("z", "zulu") + + return []Step{ + {"create a", create(publicops.CreateRequest{Issue: a})}, + {"create b blocked by a", create(publicops.CreateRequest{Issue: b, + Dependencies: []publicops.CreateDependency{{TargetID: id("a"), Type: types.DepBlocks}}})}, + {"create epic", create(publicops.CreateRequest{Issue: epic})}, + {"create epic.1 under epic", create(publicops.CreateRequest{Issue: child, ParentID: id("epic")})}, + {"create wisp w1", create(publicops.CreateRequest{Issue: w1})}, + {"create no-history nh", create(publicops.CreateRequest{Issue: nh})}, + {"create c", create(publicops.CreateRequest{Issue: c})}, + {"create d", create(publicops.CreateRequest{Issue: d})}, + {"create e", create(publicops.CreateRequest{Issue: e})}, + + {"update a fields labels metadata", update(id("a"), publicops.IssuePatch{ + Title: set("alpha two"), + Priority: set(3), + Assignee: set("bob"), + Status: set(types.StatusInProgress), + AppendNotes: set("second note"), + Labels: publicops.LabelPatch{Add: []string{"three"}, Remove: []string{"two"}}, + Metadata: publicops.MetadataPatch{Set: map[string]json.RawMessage{"k": json.RawMessage(`2`)}}, + })}, + {"update a no-op", update(id("a"), publicops.IssuePatch{Title: set("alpha two")})}, + {"update w1 to persistent", update(id("w1"), publicops.IssuePatch{Persistence: set(types.PersistenceModePersistent)})}, + {"update c to ephemeral", update(id("c"), publicops.IssuePatch{Persistence: set(types.PersistenceModeEphemeral)})}, + {"update d parent epic", update(id("d"), publicops.IssuePatch{ParentID: set(id("epic"))})}, + + {"add dependencies", func(ctx context.Context) error { + editor, err := r.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, publicops.AddDependenciesRequest{Actor: "carol", Edges: []publicops.DependencyEdge{ + {IssueID: id("d"), DependsOnID: id("b"), Type: types.DepBlocks}, + {IssueID: id("e"), DependsOnID: id("a"), Type: types.DepRelated}, + {IssueID: id("nh"), DependsOnID: id("d"), Type: types.DepBlocks}, + }}) + return err + }}, + {"remove dependency b->a", func(ctx context.Context) error { + editor, err := r.DependencyEditor() + if err != nil { + return err + } + _, err = editor.RemoveDependency(ctx, publicops.RemoveDependencyRequest{Actor: "carol", IssueID: id("b"), DependsOnID: id("a")}) + return err + }}, + {"comment on a and nh", func(ctx context.Context) error { + commenter, err := r.Commenter() + if err != nil { + return err + } + if _, err := commenter.AddComment(ctx, publicops.AddCommentRequest{Author: "carol", IssueID: id("a"), Text: "hello a"}); err != nil { + return err + } + _, err = commenter.AddComment(ctx, publicops.AddCommentRequest{Author: "carol", IssueID: id("nh"), Text: "hello nh"}) + return err + }}, + {"claim heartbeat release b", func(ctx context.Context) error { + claimer, err := r.IssueClaimer() + if err != nil { + return err + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "dave", IssueID: id("b")}); err != nil { + return fmt.Errorf("claim: %w", err) + } + heartbeater, err := r.Heartbeater() + if err != nil { + return err + } + if _, err := heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: "dave", IssueID: id("b")}); err != nil { + return fmt.Errorf("heartbeat: %w", err) + } + releaser, err := r.Releaser() + if err != nil { + return err + } + _, err = releaser.Release(ctx, publicops.ReleaseRequest{Actor: "dave", IssueID: id("b")}) + return err + }}, + {"claim e and reclaim no lease", func(ctx context.Context) error { + claimer, err := r.IssueClaimer() + if err != nil { + return err + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "erin", IssueID: id("e")}); err != nil { + return fmt.Errorf("claim: %w", err) + } + reclaimer, err := r.LeaseReclaimer() + if err != nil { + return err + } + // e's lease has not expired, so the sweep reverts nothing. + _, err = reclaimer.Reclaim(ctx, publicops.ReclaimRequest{Actor: "reaper", + Filter: publicops.ReclaimFilter{IDs: []string{id("e")}}}) + return err + }}, + {"metadata compare-and-set on a", func(ctx context.Context) error { + cas, err := r.MetadataCAS() + if err != nil { + return err + } + _, err = cas.CompareAndSetKey(ctx, publicops.CompareAndSetKeyRequest{Actor: "frank", IssueID: id("a"), Key: "cas", Value: raw(`"v1"`)}) + return err + }}, + {"close reopen close epic.1", func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + if _, err := lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("epic.1"), Reason: "done"}); err != nil { + return fmt.Errorf("close: %w", err) + } + if _, err := lifecycle.Reopen(ctx, publicops.ReopenRequest{Actor: "bob", IssueID: id("epic.1"), Reason: "not yet"}); err != nil { + return fmt.Errorf("reopen: %w", err) + } + _, err = lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("epic.1"), Reason: "done again"}) + return err + }}, + {"close batch b and refused epic", func(ctx context.Context) error { + closer, err := r.BatchCloser() + if err != nil { + return err + } + // b closes; epic keeps its open child d, so close policy refuses + // that item and the batch keeps its survivor. + _, err = closer.CloseBatch(ctx, publicops.CloseBatchRequest{Actor: "bob", Items: []publicops.BatchCloseItem{ + {IssueID: id("b"), Reason: "b done"}, {IssueID: id("epic"), Reason: "epic done"}, + }}) + return err + }}, + {"force close blocked no-history nh", func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + _, err = lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("nh"), Reason: "nh done", Force: true}) + return err + }}, + {"create batch", func(ctx context.Context) error { + creator, err := r.BatchCreator() + if err != nil { + return err + } + f1 := newIssue("f1", "foxtrot one", types.TypeTask) + f1.Labels = []string{"batch"} + f2 := newIssue("f2", "foxtrot two", types.TypeTask) + fw := newIssue("fw", "foxtrot wisp", types.TypeTask) + fw.Ephemeral = true + _, err = creator.CreateBatch(ctx, publicops.CreateBatchRequest{Actor: "gina", Items: []publicops.BatchCreateItem{ + {Issue: f1}, + {Issue: f2, Dependencies: []publicops.CreateDependency{{TargetID: id("f1"), Type: types.DepBlocks}}}, + {Issue: fw}, + }}) + return err + }}, + {"apply batch", func(ctx context.Context) error { + applier, err := r.BatchApplier() + if err != nil { + return err + } + g1 := newIssue("g1", "golf one", types.TypeTask) + g2 := newIssue("g2", "golf two", types.TypeBug) + g2.Labels = []string{"applied"} + _, err = applier.ApplyBatch(ctx, publicops.ApplyBatchRequest{Actor: "hank", Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "g1", Issue: g1}}, + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "g2", Issue: g2}}, + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{Source: publicops.Ref{Key: "g2"}, Target: publicops.Ref{Key: "g1"}, Type: types.DepBlocks}}, + {Kind: publicops.ItemUpdate, Update: &publicops.UpdateItem{Target: publicops.Ref{ID: id("d")}, Patch: publicops.IssuePatch{Assignee: set("hank")}}}, + {Kind: publicops.ItemClose, Close: &publicops.CloseItem{Target: publicops.Ref{Key: "g1"}, Reason: "applied"}}, + }}) + return err + }}, + {"defer f1 into the past and wake it", func(ctx context.Context) error { + past := fixedAt + if err := update(id("f1"), publicops.IssuePatch{Status: set(types.StatusDeferred), DeferUntil: set(&past)})(ctx); err != nil { + return fmt.Errorf("defer: %w", err) + } + return wake(ctx) + }}, + {"delete f2", func(ctx context.Context) error { + deleter, err := r.Deleter() + if err != nil { + return err + } + _, err = deleter.Delete(ctx, publicops.DeleteRequest{Actor: "ivan", IDs: []string{id("f2")}, Force: true}) + return err + }}, + + // The wisps plane, role by role. + {"create wisps x y z", func(ctx context.Context) error { + for _, w := range []*types.Issue{x, y, z} { + if err := create(publicops.CreateRequest{Issue: w})(ctx); err != nil { + return fmt.Errorf("create %s: %w", w.ID, err) + } + } + return nil + }}, + {"update wisp x fields labels metadata", update(id("x"), publicops.IssuePatch{ + Title: set("x-ray two"), + Priority: set(1), + Assignee: set("bob"), + Status: set(types.StatusInProgress), + AppendNotes: set("wisp note"), + Labels: publicops.LabelPatch{Add: []string{"xn"}, Remove: []string{"xm"}}, + Metadata: publicops.MetadataPatch{Set: map[string]json.RawMessage{"k": json.RawMessage(`2`)}}, + })}, + {"update wisp x no-op", update(id("x"), publicops.IssuePatch{Title: set("x-ray two")})}, + {"claim and heartbeat refuse wisps", func(ctx context.Context) error { + // The lease roles do not lease a wisp: Claim answers ErrNotFound, + // Heartbeat ErrNotClaimable, and neither may write. + claimer, err := r.IssueClaimer() + if err != nil { + return err + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "dave", IssueID: id("y")}); !errors.Is(err, publicops.ErrNotFound) { + return fmt.Errorf("claim wisp y = %v, want ErrNotFound", err) + } + heartbeater, err := r.Heartbeater() + if err != nil { + return err + } + if _, err := heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: "bob", IssueID: id("x")}); !errors.Is(err, publicops.ErrNotClaimable) { + return fmt.Errorf("heartbeat wisp x = %v, want ErrNotClaimable", err) + } + return nil + }}, + {"reclaim sweep scoped to in-progress wisp x", func(ctx context.Context) error { + reclaimer, err := r.LeaseReclaimer() + if err != nil { + return err + } + // The sweep only reads the issues table, so it reverts nothing. + _, err = reclaimer.Reclaim(ctx, publicops.ReclaimRequest{Actor: "reaper", + Filter: publicops.ReclaimFilter{IDs: []string{id("x")}}}) + return err + }}, + {"release wisp x", func(ctx context.Context) error { + releaser, err := r.Releaser() + if err != nil { + return err + } + _, err = releaser.Release(ctx, publicops.ReleaseRequest{Actor: "bob", IssueID: id("x")}) + return err + }}, + {"add and remove wisp dependencies", func(ctx context.Context) error { + editor, err := r.DependencyEditor() + if err != nil { + return err + } + if _, err := editor.AddDependencies(ctx, publicops.AddDependenciesRequest{Actor: "carol", Edges: []publicops.DependencyEdge{ + {IssueID: id("y"), DependsOnID: id("x"), Type: types.DepBlocks}, + {IssueID: id("x"), DependsOnID: id("a"), Type: types.DepRelated}, + }}); err != nil { + return fmt.Errorf("add: %w", err) + } + _, err = editor.RemoveDependency(ctx, publicops.RemoveDependencyRequest{Actor: "carol", IssueID: id("y"), DependsOnID: id("x")}) + return err + }}, + {"comment on wisp x", func(ctx context.Context) error { + commenter, err := r.Commenter() + if err != nil { + return err + } + _, err = commenter.AddComment(ctx, publicops.AddCommentRequest{Author: "carol", IssueID: id("x"), Text: "hello x"}) + return err + }}, + {"metadata compare-and-set on wisp x", func(ctx context.Context) error { + cas, err := r.MetadataCAS() + if err != nil { + return err + } + _, err = cas.CompareAndSetKey(ctx, publicops.CompareAndSetKeyRequest{Actor: "frank", IssueID: id("x"), Key: "cas", Value: raw(`"w1"`)}) + return err + }}, + {"close reopen wisp x", func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + if _, err := lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("x"), Reason: "x done"}); err != nil { + return fmt.Errorf("close: %w", err) + } + _, err = lifecycle.Reopen(ctx, publicops.ReopenRequest{Actor: "bob", IssueID: id("x"), Reason: "x not yet"}) + return err + }}, + {"apply batch on the wisps plane", func(ctx context.Context) error { + applier, err := r.BatchApplier() + if err != nil { + return err + } + h1 := wisp("h1", "hotel wisp") + h1.Labels = []string{"applied"} + _, err = applier.ApplyBatch(ctx, publicops.ApplyBatchRequest{Actor: "hank", Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "h1", Issue: h1}}, + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{Source: publicops.Ref{ID: id("y")}, Target: publicops.Ref{Key: "h1"}, Type: types.DepBlocks}}, + {Kind: publicops.ItemUpdate, Update: &publicops.UpdateItem{Target: publicops.Ref{ID: id("y")}, Patch: publicops.IssuePatch{Assignee: set("hank")}}}, + {Kind: publicops.ItemClose, Close: &publicops.CloseItem{Target: publicops.Ref{Key: "h1"}, Reason: "applied"}}, + }}) + return err + }}, + {"close batch wisps x and nh", func(ctx context.Context) error { + closer, err := r.BatchCloser() + if err != nil { + return err + } + // x closes; nh is already closed, so the batch closes x alone. + _, err = closer.CloseBatch(ctx, publicops.CloseBatchRequest{Actor: "bob", Items: []publicops.BatchCloseItem{ + {IssueID: id("x"), Reason: "x done again"}, {IssueID: id("nh"), Reason: "nh again"}, + }}) + return err + }}, + {"defer wisp z into the past and wake it", func(ctx context.Context) error { + past := fixedAt + if err := update(id("z"), publicops.IssuePatch{Status: set(types.StatusDeferred), DeferUntil: set(&past)})(ctx); err != nil { + return fmt.Errorf("defer: %w", err) + } + return wake(ctx) + }}, + {"delete wisp y", func(ctx context.Context) error { + deleter, err := r.Deleter() + if err != nil { + return err + } + _, err = deleter.Delete(ctx, publicops.DeleteRequest{Actor: "ivan", IDs: []string{id("y")}, Force: true}) + return err + }}, + } +} + +// DoltStorageSteps are the store legs' extra steps: the storage.DoltStorage +// methods most of bd's commands still write through, including the import +// upsert (CreateIssuesWithFullOptions) whose create-versus-update choice the +// stack rewired, promotion out of the wisps table, and RunInTransaction. +func DoltStorageSteps(s storage.DoltStorage) []Step { + return []Step{ + {"store create s1", func(ctx context.Context) error { + s1 := newIssue("s1", "sierra one", types.TypeTask) + s1.Labels = []string{"store"} + return s.CreateIssue(ctx, s1, "sam") + }}, + {"store create issues s2 s3", func(ctx context.Context) error { + return s.CreateIssues(ctx, []*types.Issue{newIssue("s2", "sierra two", types.TypeTask), newIssue("s3", "sierra three", types.TypeBug)}, "sam") + }}, + {"store import upsert s1 and new s4", func(ctx context.Context) error { + s1 := newIssue("s1", "sierra one imported", types.TypeTask) + s1.Labels = []string{"store", "imported"} + s1.UpdatedAt = fixedAt.Add(time.Hour) + s4 := newIssue("s4", "sierra four", types.TypeTask) + s4.Dependencies = []*types.Dependency{{IssueID: id("s4"), DependsOnID: id("s1"), Type: types.DepBlocks, CreatedAt: fixedAt}} + s4.Comments = []*types.Comment{{ID: "ho-imported-comment", IssueID: id("s4"), Author: "importer", Text: "imported", CreatedAt: fixedAt}} + sw := newIssue("sw", "sierra wisp", types.TypeTask) + sw.Ephemeral = true + return s.CreateIssuesWithFullOptions(ctx, []*types.Issue{s1, s4, sw}, "importer", + storage.BatchCreateOptions{SkipPrefixValidation: true}) + }}, + {"store update s2", func(ctx context.Context) error { + return s.UpdateIssue(ctx, id("s2"), map[string]interface{}{"title": "sierra two updated", "priority": 1, "assignee": "sam"}, "sam") + }}, + {"store update s2 no-op", func(ctx context.Context) error { + return s.UpdateIssue(ctx, id("s2"), map[string]interface{}{"title": "sierra two updated"}, "sam") + }}, + {"store labels", func(ctx context.Context) error { + if err := s.AddLabel(ctx, id("s2"), "added", "sam"); err != nil { + return err + } + return s.RemoveLabel(ctx, id("s1"), "store", "sam") + }}, + {"store dependencies", func(ctx context.Context) error { + if err := s.AddDependency(ctx, &types.Dependency{IssueID: id("s3"), DependsOnID: id("s2"), Type: types.DepBlocks}, "sam"); err != nil { + return err + } + return s.RemoveDependency(ctx, id("s4"), id("s1"), "sam") + }}, + {"store comment", func(ctx context.Context) error { + _, err := s.AddIssueComment(ctx, id("s3"), "sam", "store comment") + return err + }}, + {"store claim s3", func(ctx context.Context) error { + bulk, ok := s.(storage.BulkIssueStore) + if !ok { + return fmt.Errorf("%T is not a storage.BulkIssueStore", s) + } + return bulk.ClaimIssue(ctx, id("s3"), "tess") + }}, + {"store close and reopen s2", func(ctx context.Context) error { + if err := s.CloseIssue(ctx, id("s2"), "closed by store", "sam", "session-1"); err != nil { + return err + } + return s.ReopenIssue(ctx, id("s2"), "reopened by store", "sam") + }}, + {"store promote wisp sw", func(ctx context.Context) error { + bulk, ok := s.(storage.BulkIssueStore) + if !ok { + return fmt.Errorf("%T is not a storage.BulkIssueStore", s) + } + return bulk.PromoteFromEphemeral(ctx, id("sw"), "sam") + }}, + {"store transaction", func(ctx context.Context) error { + return s.RunInTransaction(ctx, "bd: historyoffequiv transaction", func(tx storage.Transaction) error { + t1 := newIssue("t1", "tango one", types.TypeTask) + if err := tx.CreateIssue(ctx, t1, "tx"); err != nil { + return err + } + if err := tx.UpdateIssue(ctx, id("t1"), map[string]interface{}{"description": "changed in tx"}, "tx"); err != nil { + return err + } + if err := tx.AddLabel(ctx, id("t1"), "tx", "tx"); err != nil { + return err + } + if err := tx.AddDependency(ctx, &types.Dependency{IssueID: id("t1"), DependsOnID: id("s1"), Type: types.DepRelated}, "tx"); err != nil { + return err + } + return tx.CloseIssue(ctx, id("s4"), "closed in tx", "tx", "") + }) + }}, + {"store delete s3", func(ctx context.Context) error { + return s.DeleteIssue(ctx, id("s3")) + }}, + } +} diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 3ade280ae..094dd2847 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -210,6 +210,7 @@ go_test( "version_history_number_admission_test.go", "version_history_preenable_scan_test.go", "version_history_test.go", + "wake_defers_test.go", "wisp_routing_test.go", ], data = glob(["testdata/**"]) + [ @@ -228,6 +229,7 @@ go_test( "//internal/storage/journalscan", "//internal/storage/rowid", "//internal/storage/sqlbuild", + "//internal/storage/sqlcount", "//internal/types", "//internal/validation", "//issueops", diff --git a/internal/storage/issueops/wake_defers_test.go b/internal/storage/issueops/wake_defers_test.go new file mode 100644 index 000000000..fae1b596e --- /dev/null +++ b/internal/storage/issueops/wake_defers_test.go @@ -0,0 +1,127 @@ +package issueops + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "slices" + "sync/atomic" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage/sqlcount" +) + +// With versioned history off the mint is a no-op, so the sweep has no reason +// to look at an expired row before it wakes it, nor after. With history on it +// reads the woken row to mint its version; a later change that also checks +// each row against the mint before waking it (so that one unversionable row +// cannot abort the whole sweep) must keep that check free when history is +// off. +// +// The mock scripts exactly the statements a one-row sweep issues: the +// expired-row SELECT, the UPDATE, the event's same-content SELECT and its +// INSERT, and the empty wisps pass. Nothing else is expected, and the mock is +// reached through sqlcount, which counts every statement the sweep sends. So +// a sweep that loaded the row (or read its participation) before or after +// waking it fails the pin even if it swallowed the error sqlmock answers that +// statement with, which sqlmock's own checks would not see. The history-on +// run of the same script is the control: the mint's read must make the pin +// refuse it. +func TestWakeExpiredDefersWithHistoryOffLoadsNothingBeforeWaking(t *testing.T) { + if err := wakeAgainstScript(t, false); err != nil { + t.Fatalf("history off: %v", err) + } + t.Run("history on is refused by the same script", func(t *testing.T) { + err := wakeAgainstScript(t, true) + if err == nil { + t.Fatal("the sweep with history on passed the history-off script; " + + "the script no longer sees the reads history adds, so the history-off run above proves nothing") + } + t.Logf("history on, as expected: %v", err) + }) +} + +// wakeStatements is how many statements the script below holds, and so how +// many the history-off sweep may send. +const wakeStatements = 5 + +// wakeAgainstScript runs a one-row sweep on a mocked transaction scoped with +// versioned history as asked, against the script of the statements the +// history-off sweep issues. It returns why the sweep broke the script, or nil +// if it sent exactly the scripted statements and woke the scripted row. +func wakeAgainstScript(t *testing.T, history bool) error { + t.Helper() + mock, tx, counts := countedMockTx(t) + if history { + t.Cleanup(ScopeVersionedHistoryTransaction(tx, true)) + } + // Otherwise tx is not scoped for versioned history, which is how every + // transaction reads while it is off. + mock.ExpectQuery(`SELECT id FROM issues WHERE status = 'deferred'`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("bd-1")) + mock.ExpectExec(`UPDATE issues SET status = 'open', defer_until = NULL`). + WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), "bd-1"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT id FROM events WHERE issue_id`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectExec(`INSERT INTO events`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT id FROM wisps WHERE status = 'deferred'`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + + got, err := WakeExpiredDefersInTx(t.Context(), tx) + if err != nil { + return fmt.Errorf("the sweep failed against the script: %w", err) + } + if n := counts.Total(); n != wakeStatements { + return fmt.Errorf("the sweep sent %d statements, want exactly the %d scripted; "+ + "sqlmock refused each extra one with an error the sweep dropped", n, wakeStatements) + } + if err := mock.ExpectationsWereMet(); err != nil { + return fmt.Errorf("the sweep did not send the scripted statements: %w", err) + } + if !slices.Equal(got.Issues, []string{"bd-1"}) || len(got.Wisps) != 0 { + return fmt.Errorf("woken = issues %v, wisps %v, want [bd-1] and none", got.Issues, got.Wisps) + } + return nil +} + +var sqlmockSeq atomic.Int64 + +// countedMockTx begins a transaction on a fresh sqlmock connection that is +// reached through sqlcount, so a pin can count every statement the code +// under test sends, including one whose error it swallowed. +func countedMockTx(t *testing.T) (sqlmock.Sqlmock, *sql.Tx, *sqlcount.Counts) { + t.Helper() + dsn := fmt.Sprintf("issueops_counted_%d", sqlmockSeq.Add(1)) + mockDB, mock, err := sqlmock.NewWithDSN(dsn) + if err != nil { + t.Fatalf("sqlmock.NewWithDSN: %v", err) + } + counts := &sqlcount.Counts{} + db := sql.OpenDB(sqlcount.WrapConnector(sqlmockConnector{dsn: dsn, driver: mockDB.Driver()}, counts)) + t.Cleanup(func() { + _ = db.Close() + _ = mockDB.Close() + }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + t.Cleanup(func() { _ = tx.Rollback() }) + return mock, tx, counts +} + +// sqlmockConnector opens the sqlmock connection registered under dsn, for +// sql.OpenDB through a wrapping connector. +type sqlmockConnector struct { + dsn string + driver driver.Driver +} + +func (c sqlmockConnector) Connect(context.Context) (driver.Conn, error) { return c.driver.Open(c.dsn) } +func (c sqlmockConnector) Driver() driver.Driver { return c.driver } diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 96c3d988c..19101d195 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -125,6 +125,7 @@ go_test( "external_doltserver_provider_test.go", "graph_counter_contract_test.go", "heartbeater_contract_test.go", + "history_off_differential_test.go", "importer_contract_test.go", "importer_uow_test.go", "issue_claimer_test.go", @@ -198,6 +199,7 @@ go_test( "//internal/storage/dbproxy/server", "//internal/storage/domain", "//internal/storage/domain/db", + "//internal/storage/historyoffequiv", "//internal/storage/issueops", "//internal/storage/schema", "//internal/testutil", diff --git a/internal/storage/uow/history_off_differential_test.go b/internal/storage/uow/history_off_differential_test.go new file mode 100644 index 000000000..aabe0eefe --- /dev/null +++ b/internal/storage/uow/history_off_differential_test.go @@ -0,0 +1,92 @@ +package uow + +import ( + "context" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/historyoffequiv" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestVersionedHistoryOffMatchesPreStackGolden_UOW is the unit-of-work leg's +// versioned-history flag-off differential (internal/storage/historyoffequiv): +// on a provider whose versioned history was never turned on, the scripted +// mutation set must leave every table as the code before the +// versioned-history stack left it, apart from that stack's new columns and +// table staying NULL and empty. +// +// Beyond the shared role scenario this leg drives the domain create that +// leaves CreateOnly unset (the quick, todo and batch proxied servers' path): +// it is an upsert, and the stack gave it an existence probe that must not run +// while history is off (see TestInsertUpsertWithHistoryOffIssuesThePreStackStatements +// in domain/db). The importer role's upsert of an existing row runs too. +func TestVersionedHistoryOffMatchesPreStackGolden_UOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, historyoffequiv.Prefix) + roles, ok := provider.(historyoffequiv.Roles) + if !ok { + t.Fatalf("provider %T does not offer every role the scenario drives", provider) + } + fixedAt := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) + issue := func(id, title string) *types.Issue { + return &types.Issue{ID: id, Title: title, Description: "about " + id, Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, CreatedAt: fixedAt, UpdatedAt: fixedAt} + } + domainCreate := func(wisp bool, is *types.Issue) func(context.Context) error { + return func(ctx context.Context) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + params := domain.CreateIssueParams{Issue: is, ExplicitID: is.ID} + var err error + if wisp { + _, err = uw.IssueUseCase().CreateWisp(ctx, params, "uma") + } else { + _, err = uw.IssueUseCase().CreateIssue(ctx, params, "uma") + } + return "bd: create " + is.ID, err + }) + } + } + historyoffequiv.Run(t, historyoffequiv.Leg{ + Name: "uow", + Roles: roles, + Wake: func(ctx context.Context) error { + _, err := WakeExpiredDefers(ctx, provider) + return err + }, + Extra: []historyoffequiv.Step{ + {Name: "domain upsert new u1", Run: domainCreate(false, issue("ho-u1", "uniform one"))}, + {Name: "domain upsert existing u1", Run: domainCreate(false, issue("ho-u1", "uniform one again"))}, + {Name: "domain upsert new wisp uw", Run: domainCreate(true, issue("ho-uw", "uniform wisp"))}, + {Name: "domain upsert existing wisp uw", Run: domainCreate(true, issue("ho-uw", "uniform wisp again"))}, + {Name: "import upsert u1 and new u2", Run: func(ctx context.Context) error { + importer, err := provider.(interface { + Importer() (publicops.Importer, error) + }).Importer() + if err != nil { + return err + } + u1 := issue("ho-u1", "uniform one imported") + u1.UpdatedAt = fixedAt.Add(time.Hour) + u2 := issue("ho-u2", "uniform two") + u2.Labels = []string{"imported"} + u2.Dependencies = []*types.Dependency{{IssueID: "ho-u2", DependsOnID: "ho-u1", Type: types.DepBlocks, CreatedAt: fixedAt}} + u2.Comments = []*types.Comment{{ID: "ho-u2-comment", IssueID: "ho-u2", Author: "importer", Text: "imported", CreatedAt: fixedAt}} + _, err = importer.ImportBatch(ctx, publicops.ImportBatchRequest{Actor: "importer", + Issues: []*types.Issue{u1, u2}, SkipPrefixValidation: true}) + return err + }}, + }, + Query: func(ctx context.Context, query string) ([][]*string, error) { + return RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) ([][]*string, error) { + result, err := uw.RawSQLUseCase().Query(ctx, query) + if err != nil { + return nil, err + } + return historyoffequiv.AnyRows(result.Rows), nil + }) + }, + }) +} diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 410728ae0..73e966ddc 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -259,6 +259,7 @@ release_cross_build( "//internal/storage/externaldeps:externaldeps", "//internal/storage/fs:fs", "//internal/storage/git:git", + "//internal/storage/historyoffequiv:historyoffequiv", "//internal/storage/issueops:issueops", "//internal/storage/journalscan:journalscan", "//internal/storage/kvkeys:kvkeys", From 8fac20a7f1ae0d5941094516bb49b66056755656 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 04:27:28 +0000 Subject: [PATCH 20/54] test(oracle-a): normalize per-run revision token and workspace path so main-vs-main is clean (#7680) * test(oracle-a): normalize the per-write revision token so main-vs-main is clean Since #5535, `bd show --json` publishes `"revision"`, the decimal spelling of the row's row_lock. Every create and status/ownership write remints that token from crypto/rand (issueops.freshRowLock), so it differs between any two runs of the same scenario. The harness did not normalize it, so Oracle A failed against itself: on the curated tier, origin/main vs origin/main scored 31 pass / 13 fail in scope, and each failure was a `show --json` step that differed only in `"revision"`. normalize() now collapses a non-zero decimal `"revision"` string to ``. It keeps everything else literal: the member name, its presence and its string type, "0" (a row that was never stamped), number-typed values, other `*_revision` members, and escaped text inside a JSON string. A new unit test covers each of these cases. Curated tier, REF_REF=origin/main, run against itself: before 31 pass / 13 fail in scope (0/1 out of scope); after 44 pass / 0 fail (1/0). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(oracle-a): normalize the per-run workspace path Each scenario runs in a fresh tempfile::tempdir(), so any bd output that echoes the workspace path differs on every run. When origin/main's bd was captured twice with the deep tier (ORACLE_CATALOG=1, 546 scenarios), 40 catalog scenarios differed only in that path. All 40 start with their own `init --prefix X --non-interactive` step, which runs in a workspace the harness already initialized and fails with "Found existing Dolt database: /tmp/.tmpXXXXXX/.beads/embeddeddolt/X". run_scenario now hands normalize_in_workspace() the exact workspace path and its symlink-resolved spelling, and that function replaces only those strings with , longest first. Other paths stay literal, so no stable value is collapsed. All 40 scenarios are out of scope, so in-scope counts do not change. Deep tier, origin/main's bd against itself, scored by the scoreboard: - base harness: 256 pass / 67 fail in scope, 154 / 69 out of scope. - with both normalizers: 323 / 0 in scope, 219 / 4 out of scope. The out-of-scope remainder changes from run to run and none of it is a field value. It comes from randomized "Tip:" lines in human output and one unordered `list --format dot`. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(oracle-a): document and pin the match scope and the workspace wiring (mc-ro9kg) Review fixes for #7680, iteration 1. revision_regex runs over the raw stdout/stderr text before any JSON parse, so it collapses every "revision" member holding a non-zero decimal string, at any depth and in plain text, not only the bd show --json issue token. That includes a user metadata key named revision, which the in-scope --set-metadata/--metadata flags can set, so a change to its value passes the gate. The regex comment and a new README "What green does NOT prove" bullet now say so, and normalize_collapses_revision_by_member_name_at_any_depth pins the scope: a nested metadata member and plain text are collapsed, while a metadata key with any other name still gates. The old test name said nothing else was collapsed; it is now normalize_collapses_non_zero_revision_strings. run_scenario_collapses_the_workspace_in_both_step_streams runs run_scenario against a stub bd that prints its cwd on stdout and its resolved path on stderr, and expects in both. Before, the two normalize_in_workspace calls in run_scenario could be reverted with every unit test and the curated tier still green. The remint wording in the comment and the README now names every writer (create, update, close, reopen, claim and unclaim) instead of only status/ownership writes. Tests: cargo test --locked --offline (7 passed), clippy clean, no new rustfmt hunks. Mutants, each in its own copy and target dir: reverting either or both normalize_in_workspace calls fails the run_scenario test; skipping a nested metadata member, or matching only after { or ",", fails the scope test. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- tests/oracle-a/README.md | 21 ++- tests/oracle-a/harness/src/differential.rs | 181 ++++++++++++++++++++- tests/oracle-a/run-oracle-a.sh | 6 +- 3 files changed, 195 insertions(+), 13 deletions(-) diff --git a/tests/oracle-a/README.md b/tests/oracle-a/README.md index 9ce8c5748..3eafcb558 100644 --- a/tests/oracle-a/README.md +++ b/tests/oracle-a/README.md @@ -15,9 +15,11 @@ Each scenario is an ordered list of `bd` CLI argv steps run as real processes in a throwaway workspace (`bd init` + steps). For every step the harness compares **exit code**, **stderr**, and **JSON-aware stdout** (object key order ignored; array order compared as a multiset by default, as an ordered sequence for -scenarios flagged `ordered`). Volatile values — timestamps, UUIDs, and the host -actor identity — are normalized to ``/``/``/`` before -comparison. **Tolerated in-scope divergences: zero.** +scenarios flagged `ordered`). Volatile values — timestamps, UUIDs, the host +actor identity, the `bd show --json` `"revision"` token (matched by member name, +see below), and the run's own throwaway workspace path — are normalized to +``/``/``/``/``/`` before comparison. +**Tolerated in-scope divergences: zero.** ## Run it @@ -112,7 +114,18 @@ the loop — every scenario uses embedded Dolt in its own tempdir. - **Anything invisible to stdout/stderr/exit diffing.** Post-commit hook firing, rollback-suppressed side effects, and N-commits-instead-of-one are deliberately NOT asserted here. Timestamp *values* are erased by `` normalization, so a - local-vs-UTC divergence in a timestamp value is also out of reach. + local-vs-UTC divergence in a timestamp value is also out of reach. Likewise a + non-zero `"revision"` token is erased to `` (every create, update, close, + reopen, claim and unclaim remints it from crypto/rand), so whether a write + remints it is not gated; the member's presence, its string type, and a + never-stamped `"0"` still are. +- **`` matches a member name, not a position.** The `"revision"` collapse runs + over the raw stdout/stderr text before any JSON parse, so it erases EVERY + `"revision"` member holding a non-zero decimal string — at any depth, and in + plain (non-JSON) text too — not only the `bd show --json` issue token. That + includes a user metadata key named `revision`, which the in-scope + `--set-metadata`/`--metadata` flags can set: a change to its value passes the + gate. No scenario sets one today; use another key name in any that would. - **The scenario environment is scrubbed, and `BEADS_TEST_MODE=1` is ungated.** Scenarios run with an explicit env whitelist (`PATH`/`HOME`/`TMPDIR` + `BEADS_TEST_MODE=1`) rather than the inherited host env, so a green certifies the diff --git a/tests/oracle-a/harness/src/differential.rs b/tests/oracle-a/harness/src/differential.rs index cf0078190..1ab47cdf7 100644 --- a/tests/oracle-a/harness/src/differential.rs +++ b/tests/oracle-a/harness/src/differential.rs @@ -89,20 +89,72 @@ fn uuid_regex() -> &'static regex::Regex { }) } -/// Replace volatile values (timestamps, host identity, random UUIDs) with stable -/// tokens. UUIDs (e.g. comment ids) are random per run, so byte-diffing them is -/// meaningless — collapse them like timestamps. +fn revision_regex() -> &'static regex::Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + // The `"revision": ""` JSON member (types.IssueDetails.Revision, on + // `bd show --json` since #5535). The token is the row's row_lock, which + // every create, update, close, reopen, claim and unclaim remints from + // crypto/rand (issueops.freshRowLock), so its value differs on every run. + // Only a NON-ZERO decimal token is collapsed: "0" (a never-stamped row) + // stays literal, and the member's name, presence and string type are + // untouched. + // + // The match is by member NAME over the raw text, before any JSON parse, + // so it collapses EVERY `"revision"` member holding such a string, at any + // depth and in plain (non-JSON) text too, not only the issue token. That + // includes a user metadata key named `revision`, which the in-scope + // --set-metadata/--metadata flags can set: a change to its value would + // pass the gate. No scenario sets one; use another key name in any that + // would. Pinned by normalize_collapses_revision_by_member_name_at_any_depth. + regex::Regex::new(r#"("revision"\s*:\s*")-?[1-9][0-9]*(")"#).expect("valid revision regex") + }) +} + +/// Replace volatile values (timestamps, host identity, random UUIDs, revision +/// tokens) with stable tokens. UUIDs (e.g. comment ids) and revision tokens are +/// random per run, so byte-diffing them is meaningless — collapse them like +/// timestamps. pub fn normalize(s: &str) -> String { - let s = ts_regex().replace_all(s, "").into_owned(); + let s = revision_regex() + .replace_all(s, "${1}${2}") + .into_owned(); + let s = ts_regex().replace_all(&s, "").into_owned(); let s = uuid_regex().replace_all(&s, "").into_owned(); s.replace(ACTOR_EMAIL, "").replace(ACTOR_NAME, "") } +/// `normalize`, after first collapsing this run's own workspace directory to +/// ``. Each run gets a fresh random tempdir, so any output that +/// echoes it (e.g. the "Found existing Dolt database: /.beads/..." error a +/// scenario's own `init` step hits) differs on every run. Only these exact path +/// strings are replaced, longest first, so a spelling that contains another +/// (`/private/var/...` around `/var/...`) is replaced whole. +pub fn normalize_in_workspace(s: &str, workspace: &[String]) -> String { + let mut paths: Vec<&str> = workspace + .iter() + .map(String::as_str) + .filter(|p| !p.is_empty()) + .collect(); + paths.sort_by_key(|p| std::cmp::Reverse(p.len())); + let mut s = s.to_string(); + for p in paths { + s = s.replace(p, ""); + } + normalize(&s) +} + /// Run a scenario against `bin` in a throwaway workspace, returning the /// normalized trace. Requires the binary to support `init -p --quiet` /// and to honor the workspace cwd (real bd uses embedded Dolt under `.beads/`). pub fn run_scenario(bin: &Path, sc: &Scenario) -> std::io::Result { let work = tempfile::tempdir()?; + // Every spelling of the workspace bd may print: the tempdir as created and + // its symlink-resolved form (macOS tempdirs sit under /var -> /private/var). + let mut workspace = vec![work.path().to_string_lossy().into_owned()]; + if let Ok(real) = work.path().canonicalize() { + workspace.push(real.to_string_lossy().into_owned()); + } let run = |args: &[String]| -> std::io::Result { // Scrub the environment to an explicit whitelist. The upstream harness // inherited the FULL host env, so any host BEADS_*/BD_* var @@ -142,8 +194,8 @@ pub fn run_scenario(bin: &Path, sc: &Scenario) -> std::io::Result { steps.push(StepResult { args: args.clone(), exit: out.status.code().unwrap_or(-1), - stdout: normalize(&String::from_utf8_lossy(&out.stdout)), - stderr: normalize(&String::from_utf8_lossy(&out.stderr)), + stdout: normalize_in_workspace(&String::from_utf8_lossy(&out.stdout), &workspace), + stderr: normalize_in_workspace(&String::from_utf8_lossy(&out.stderr), &workspace), }); } Ok(Trace { @@ -325,6 +377,123 @@ mod tests { assert!(!n.contains("2026")); } + #[test] + fn normalize_collapses_non_zero_revision_strings() { + // Two runs of the same `show --json`: only the minted token differs. + let run1 = + "[\n {\n \"id\": \"g-root\",\n \"revision\": \"-1228122709301203494\"\n }\n]\n"; + let run2 = + "[\n {\n \"id\": \"g-root\",\n \"revision\": \"5267907219149755461\"\n }\n]\n"; + assert_eq!(normalize(run1), normalize(run2)); + assert!(normalize(run1).contains(r#""revision": """#)); + assert!(stdout_equiv(&normalize(run1), &normalize(run2), true)); + assert_eq!(normalize(r#"{"revision":"42"}"#), r#"{"revision":""}"#); + + // Everything that is NOT a minted token keeps its literal value: + // "0" (a never-stamped row) must still diverge from a minted token. + assert_eq!(normalize(r#"{"revision":"0"}"#), r#"{"revision":"0"}"#); + assert!(!stdout_equiv( + &normalize(r#"{"revision":"0"}"#), + &normalize(r#"{"revision":"7"}"#), + false + )); + // a number-typed member (a wire-type change) is not collapsed, + assert_eq!(normalize(r#"{"revision":42}"#), r#"{"revision":42}"#); + // nor are other members whose names merely contain "revision", + assert_eq!( + normalize(r#"{"local_revision":"3"}"#), + r#"{"local_revision":"3"}"# + ); + // nor the same text quoted inside a JSON string value. + let quoted = r#"{"description":"see \"revision\": \"12\""}"#; + assert_eq!(normalize(quoted), quoted); + + // Dropping the member is still a divergence. + assert!(!stdout_equiv( + &normalize(r#"{"id":"a","revision":"9"}"#), + &normalize(r#"{"id":"a"}"#), + false + )); + } + + #[test] + fn normalize_collapses_revision_by_member_name_at_any_depth() { + // The scope documented on revision_regex: the match is by member name over + // the raw text, not by position, so a user metadata key named "revision" + // is collapsed too and a change to its value is NOT a divergence. + let meta3 = r#"[{"id":"g-1","revision":"11","metadata":{"revision":"3"}}]"#; + let meta4 = r#"[{"id":"g-1","revision":"22","metadata":{"revision":"4"}}]"#; + assert_eq!( + normalize(meta3), + r#"[{"id":"g-1","revision":"","metadata":{"revision":""}}]"# + ); + assert!(stdout_equiv(&normalize(meta3), &normalize(meta4), false)); + // Plain (non-JSON) text is matched as well. + assert_eq!( + normalize(r#"config "revision": "12" pinned"#), + r#"config "revision": "" pinned"# + ); + // A metadata key with any other name keeps its value, so it still gates. + assert!(!stdout_equiv( + &normalize(r#"{"metadata":{"rev":"3"}}"#), + &normalize(r#"{"metadata":{"rev":"4"}}"#), + false + )); + } + + #[test] + fn normalize_in_workspace_collapses_only_this_runs_workspace() { + // Listed short-first: the longer spelling must still be replaced whole. + let ws = vec![ + "/var/t/.tmpAbC123".to_string(), + "/private/var/t/.tmpAbC123".to_string(), + ]; + let err = "Found existing Dolt database: /private/var/t/.tmpAbC123/.beads/embeddeddolt/c\n"; + assert_eq!( + normalize_in_workspace(err, &ws), + "Found existing Dolt database: /.beads/embeddeddolt/c\n" + ); + // Two runs that differ only in their random workspace normalize equal. + let other = vec!["/var/t/.tmpXyZ789".to_string()]; + assert_eq!( + normalize_in_workspace("db at /var/t/.tmpAbC123/.beads", &ws), + normalize_in_workspace("db at /var/t/.tmpXyZ789/.beads", &other) + ); + // A path that is not this run's workspace stays literal. + let foreign = "db at /var/t/.tmpXyZ789/.beads"; + assert_eq!(normalize_in_workspace(foreign, &ws), foreign); + // The volatile-value normalizers still apply. + assert_eq!( + normalize_in_workspace(r#"{"revision":"5"}"#, &ws), + r#"{"revision":""}"# + ); + } + + #[cfg(unix)] + #[test] + fn run_scenario_collapses_the_workspace_in_both_step_streams() { + // A stub bd that echoes its cwd on stdout and its symlink-resolved form + // on stderr, like the "Found existing Dolt database" error: both must + // reach the trace as , so the run's random tempdir never does. + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let bin = dir.path().join("bd"); + std::fs::write( + &bin, + "#!/bin/sh\npwd\necho \"Found existing Dolt database: $(pwd -P)/.beads/x\" >&2\n", + ) + .unwrap(); + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); + let sc = Scenario::new("wiring", "w", &[&["show", "--json"]]); + let trace = run_scenario(&bin, &sc).unwrap(); + assert_eq!(trace.steps.len(), 1); + assert_eq!(trace.steps[0].stdout, "\n"); + assert_eq!( + trace.steps[0].stderr, + "Found existing Dolt database: /.beads/x\n" + ); + } + #[test] fn stdout_equiv_ignores_key_and_array_order_but_catches_set_diffs() { // key order ignored diff --git a/tests/oracle-a/run-oracle-a.sh b/tests/oracle-a/run-oracle-a.sh index b896611bc..738119abc 100755 --- a/tests/oracle-a/run-oracle-a.sh +++ b/tests/oracle-a/run-oracle-a.sh @@ -5,9 +5,9 @@ # REFERENCE bd — built from the merge base of HEAD and origin/main (the "before") # CANDIDATE bd — built from the current working tree (the "after") # and diffs each step (exit code, stderr, JSON-aware stdout) with volatile -# normalization (///). ZERO in-scope divergences is the -# gate: any in-scope FAIL means the change altered a user-visible bd behavior on -# the covered contract surface. +# normalization (/////). ZERO in-scope +# divergences is the gate: any in-scope FAIL means the change altered a +# user-visible bd behavior on the covered contract surface. # # The differential is driven by the Rust conformance harness under harness/ # (see harness/PROVENANCE.md); the harness is pointed at two Go bd binaries and From e24940e0c1f8ac3835ca39b610ddc670541d875f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 04:31:00 +0000 Subject: [PATCH 21/54] fix(workspacegate): strict --readonly never creates workspace gate files (#7672) * fix(workspacegate): strict --readonly never creates workspace gate files Strict --readonly is mutation-free by policy (effectiveRootStorePolicy), but the command chokepoint took its SHARED workspace gates with O_CREATE|O_RDWR. On a workspace no bd command had gated yet, a strict readonly command therefore left /.beads.gate.lock and the physical-root gate file (.beads/embeddeddolt.gate.lock or .beads/dolt.gate.lock) behind. Add workspacegate.Options.ReadOnly. A Shared acquisition with it opens the gate file O_RDONLY (the idiom ExclusiveHolder already uses) and never creates it. A gate file that does not exist means nothing has ever been gated there, so Acquire succeeds with an empty handle whose Release is a no-op. An existing gate file is held SHARED as before, so maintenance still sees the command and a live exclusive holder still makes it wait and then refuses it. The option is ignored for Exclusive: maintenance has to be able to create the file it locks. ReadOnly composes with the fair, bounded waiting from #7225 without changes to it: the writer-fairness probe (ExclusiveQueued) already opens the intent file read-only and treats a missing one as nothing queued, so a waiting ReadOnly acquisition creates no .intent or .info file either. readonly_wait_test.go pins that seam. The chokepoint passes ReadOnly: readonlyMode in the SHARED branch. Every other command keeps creating the gate files, which is what makes it visible to maintenance. Residual, documented at the chokepoint: on a workspace that has never been gated, a strict readonly command is invisible to a maintenance operation that starts while it runs. Ported from the versioned-beads integration fork (donnabox): 0c86e2b205 (red tests), bac6ff95dc (fix), b7a4f96a5a (fair-wait tests), 2af5a664dc (BUILD.bazel registration), and the #7225 conflict resolution in 9d128f8c51. The fork's `--readonly backup restore` refusal (6ac79d6207) is a separate change and is not included. Co-authored-by: Jim Wordelman Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * fix(cli): release the previous command's workspace gate before acquiring acquireCommandWorkspaceGates stored its new handle over whatever workspaceGateHandle already held. cobra skips PersistentPostRunE when RunE fails, so in a process that runs more than one command (tests, embedders) a command could start with the previous command's handle still set. Two things then went wrong: - on the paths that take no new gate (no .beads on disk, a failed or fail-open acquisition) the stale handle stayed set, so the previous workspace stayed held for the whole command; - on a successful acquisition the stale handle was overwritten without being released, so its flock stayed held on the previous workspace until the garbage collector closed the file, and a maintenance operation there was turned away by a command that had finished. Release any held handle first. In a single-command bd process the handle is always nil at that point, so this is a no-op there. Ported from the versioned-beads integration fork (donnabox): 3a25b42ed1 (cherry-pick of ad87442c92). The test TestAcquireCommandWorkspaceGatesReleasesThePreviousCommandsHold is new here and covers the successful-acquisition case the fork's tests do not. Co-authored-by: Donna Box Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * fix(cli): close a stale store before the leading gate release Review fixups for #7672, apply-fixes iteration 1. The leading releaseWorkspaceGates() that the second commit added to the chokepoint was the only production release without a closeStoreBeforeGateRelease() first. When an in-process command's RunE fails, Cobra skips PersistentPostRunE. The next command in the same process then dropped that command's gates while its store and provider were still open, and later overwrote and leaked them. Close them first, as every other release path does. The current command assigns store/uowProvider only after the chokepoint, so anything set there belongs to the previous command. Only multi-command processes (tests, embedders) are affected; production runs one command per process. A strict --readonly hold on a never-gated workspace locks nothing. The chokepoint still exported this PID in BEADS_GATE_SHARED_HOLDER_PID, so bd children skipped the writer-fairness queue behind a parent that blocks no exclusive acquirer. Add Handle.Empty and MultiHandle.Empty, and advertise only when at least one gate is actually locked. Tests: - A regression test runs the chokepoint over a stub store and provider. It pins that both close while the previous command's gate is still held. - A marker test forces advertiseSharedHoldEnabled on. It pins that a hold of nothing is not advertised, and that the hold is advertised once the gate files exist. - The strict-readonly chokepoint subtests now require a handle that holds nothing. SHARED fail-open leaves no handle, so these subtests now fail the review's W2 mutant, which passed them before. - A package test covers Empty. The posture comment now says the gate follows the strict flag only. Allowlisted reads and --dry-run/--inspect previews still create the gate files. Not changed: `bd --readonly backup restore` still creates gate files on its exclusive acquisition. This PR covers shared gates only. Sibling #7673 refuses a strict-readonly restore before it takes the exclusive gate. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Jim Wordelman Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Donna Box Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 + cmd/bd/workspace_gate.go | 26 +- cmd/bd/workspace_gate_test.go | 333 ++++++++++++++++++- internal/workspacegate/BUILD.bazel | 5 +- internal/workspacegate/gate.go | 51 ++- internal/workspacegate/gate_test.go | 178 ++++++++++ internal/workspacegate/readonly_wait_test.go | 183 ++++++++++ 7 files changed, 781 insertions(+), 4 deletions(-) create mode 100644 internal/workspacegate/readonly_wait_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dbfca8b0..ea893eb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -763,6 +763,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Strict `--readonly` no longer creates workspace gate files.** A + `bd --readonly` command took its shared workspace gates with + create-if-missing, so on a workspace no bd command had gated yet it left + `/.beads.gate.lock` and the physical-root gate file + (`.beads/embeddeddolt.gate.lock` or `.beads/dolt.gate.lock`) behind, although + strict readonly promises not to mutate the workspace. It now opens an + existing gate file read-only and holds it shared as before, so a maintenance + operation still sees it and a running one still makes it wait; when no gate + file exists it holds nothing. Every other command still creates the files. - **`bd init --server --external` no longer starts a Dolt server of its own.** Init builds its store config by hand and turned auto-start on for every server init, so an `--external` init that named no port launched a diff --git a/cmd/bd/workspace_gate.go b/cmd/bd/workspace_gate.go index e4ed66b22..acb54e39b 100644 --- a/cmd/bd/workspace_gate.go +++ b/cmd/bd/workspace_gate.go @@ -48,6 +48,17 @@ import ( // so would treating an interrupted wait as fail-open. // - EXCLUSIVE failures of any kind are hard errors: maintenance refuses // rather than pretends. +// - Strict --readonly is mutation-free (effectiveRootStorePolicy), so its +// SHARED hold opens the gate files read-only and never creates them (see +// workspacegate.Options.ReadOnly). A gate file that does not exist means +// nothing has ever been gated there, so the command proceeds holding +// nothing; an existing one is held SHARED like any other command's. +// Residual: on a workspace that has never been gated, a strict readonly +// command is invisible to a maintenance operation that starts while it +// runs. Every other command keeps creating the files, which is what makes +// it visible. The gate follows the strict flag only: allowlisted reads +// and --dry-run/--inspect previews also open the store read-only, but +// they keep creating the files, so they stay visible too. // // Known residual (PR-B2 scope, documented honestly): the pre-chokepoint // DISCOVERY code paths (configfile.Load at main.go's early config probe and @@ -389,6 +400,13 @@ func buildWorkspaceGateSet(beadsDir string, extraRoots ...string) ([]workspacega // workspace is selected. It stores the handle in workspaceGateHandle on // success; on the fail-open paths it returns nil with no handle. func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beadsDir string) error { + // A process can execute more than one command in tests and embedded uses. + // Never carry the preceding command's handle into an absent-workspace or + // failed acquisition path, where no new gate is held. Gates outlive the + // store, so close whatever store the preceding command left open before + // its gates drop; this command opens its own only after this point. + closeStoreBeforeGateRelease() + releaseWorkspaceGates() exclusive := commandNeedsExclusiveGate(cmd) // No workspace on disk: nothing to guard, and the store-open path will @@ -423,6 +441,9 @@ func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beads opts = workspacegate.Options{ Wait: sharedWait, IgnoreQueuedExclusive: workspacegate.InheritedSharedHold(), + // readonlyMode is already resolved here. Strict readonly promises not to + // mutate the workspace, so its SHARED hold must not create gate files. + ReadOnly: readonlyMode, } if sharedWait > 0 { opts.OnWait, stopNotice = delayedGateNotice() @@ -472,7 +493,10 @@ func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beads return nil } workspaceGateHandle = h - if !exclusive { + // A strict readonly hold on a never-gated workspace locks nothing, so it + // blocks no exclusive acquirer and gives a child nothing to stand behind: + // advertising it would let the child skip the queue for no reason. + if !exclusive && !h.Empty() { advertiseSharedHold() } return nil diff --git a/cmd/bd/workspace_gate_test.go b/cmd/bd/workspace_gate_test.go index ab85176fe..b55106a3b 100644 --- a/cmd/bd/workspace_gate_test.go +++ b/cmd/bd/workspace_gate_test.go @@ -4,7 +4,9 @@ import ( "context" "errors" "os" + "os/exec" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -13,6 +15,8 @@ import ( "time" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/workspacegate" ) @@ -93,6 +97,10 @@ func TestAcquireCommandWorkspaceGatesAbsentWorkspace(t *testing.T) { t.Cleanup(releaseWorkspaceGates) list := &cobra.Command{Use: "list"} + before := newGateTestWorkspace(t) + if err := acquireCommandWorkspaceGates(context.Background(), list, before); err != nil || workspaceGateHandle == nil { + t.Fatalf("initial command must hold a gate: %v", err) + } missing := filepath.Join(t.TempDir(), "nope", ".beads") if err := acquireCommandWorkspaceGates(context.Background(), list, missing); err != nil { t.Fatalf("absent beadsDir must be silently ungated, got %v", err) @@ -102,6 +110,109 @@ func TestAcquireCommandWorkspaceGatesAbsentWorkspace(t *testing.T) { } } +// A process that runs more than one command (tests, embedders) must not keep +// the previous command's SHARED hold once the next command takes its own +// gates: otherwise the first workspace stays held, and a maintenance +// operation there is turned away by a command that has already finished. +func TestAcquireCommandWorkspaceGatesReleasesThePreviousCommandsHold(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + list := &cobra.Command{Use: "list"} + ctx := context.Background() + + first := newGateTestWorkspace(t) + if err := acquireCommandWorkspaceGates(ctx, list, first); err != nil || workspaceGateHandle == nil { + t.Fatalf("first command must hold a gate: %v", err) + } + second := newGateTestWorkspace(t) + if err := acquireCommandWorkspaceGates(ctx, list, second); err != nil || workspaceGateHandle == nil { + t.Fatalf("second command must hold a gate: %v", err) + } + + gate, err := workspacegate.ForWorkspace(first) + if err != nil { + t.Fatal(err) + } + holder, err := gate.Acquire(ctx, workspacegate.Exclusive, workspacegate.Options{Reason: "test maintenance"}) + if err != nil { + t.Fatalf("exclusive acquire on the first command's workspace once the second command holds its own gates: %v, want success", err) + } + _ = holder.Release() +} + +// gateCloseProbeStore and gateCloseProbeProvider stand in for the store and +// provider a previous command left open; Close runs the test's probe. +type gateCloseProbeStore struct { + storage.DoltStorage + onClose func() +} + +func (s *gateCloseProbeStore) Close() error { + s.onClose() + return nil +} + +type gateCloseProbeProvider struct { + uow.UnitOfWorkProvider + onClose func() +} + +func (p *gateCloseProbeProvider) Close(context.Context) error { + p.onClose() + return nil +} + +// When a command's RunE fails, Cobra skips PersistentPostRunE, so the store it +// opened is still open when the same process runs its next command. Before the +// next chokepoint drops the previous command's gates it must close that store +// and provider: gates outlive the store, or an exclusive maintenance operation +// could start against storage that has not quiesced. +func TestAcquireCommandWorkspaceGatesClosesThePreviousCommandsStoreFirst(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + originalStore, originalProvider := store, uowProvider + t.Cleanup(func() { store, uowProvider = originalStore, originalProvider }) + list := &cobra.Command{Use: "list"} + ctx := context.Background() + + first := newGateTestWorkspace(t) + if err := acquireCommandWorkspaceGates(ctx, list, first); err != nil || workspaceGateHandle == nil { + t.Fatalf("first command must hold a gate: %v", err) + } + gate, err := workspacegate.ForWorkspace(first) + if err != nil { + t.Fatal(err) + } + // Each close records whether the first command's gate was still held. + var closes []string + probe := func(what string) func() { + return func() { + h, err := gate.Acquire(ctx, workspacegate.Exclusive, workspacegate.Options{}) + switch { + case errors.Is(err, workspacegate.ErrBusy): + closes = append(closes, what+" closed under the gate") + case err == nil: + _ = h.Release() + closes = append(closes, what+" closed after the gate dropped") + default: + t.Errorf("probing the first command's gate when its %s closed: %v", what, err) + } + } + } + store = &gateCloseProbeStore{onClose: probe("store")} + uowProvider = &gateCloseProbeProvider{onClose: probe("provider")} + + if err := acquireCommandWorkspaceGates(ctx, list, newGateTestWorkspace(t)); err != nil { + t.Fatalf("second command: %v", err) + } + if want := []string{"provider closed under the gate", "store closed under the gate"}; !slices.Equal(closes, want) { + t.Fatalf("the previous command's storage: %v, want %v", closes, want) + } + if store != nil || uowProvider != nil { + t.Fatal("the previous command's store and provider must not survive into the next command") + } +} + // holdWorkspaceGateExclusive takes beadsDir's workspace gate exclusively, // as a maintenance operation (init/restore/migrate) would. func holdWorkspaceGateExclusive(t *testing.T, beadsDir string) *workspacegate.Handle { @@ -134,11 +245,15 @@ func countGateNotices(t *testing.T) *atomic.Int32 { func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { resetGateTestEnv(t) t.Cleanup(releaseWorkspaceGates) + list := &cobra.Command{Use: "list"} + prior := newGateTestWorkspace(t) + if err := acquireCommandWorkspaceGates(context.Background(), list, prior); err != nil || workspaceGateHandle == nil { + t.Fatalf("initial command must hold a gate: %v", err) + } beadsDir := newGateTestWorkspace(t) t.Setenv(sharedGateWaitEnv, "300ms") holdWorkspaceGateExclusive(t, beadsDir) - list := &cobra.Command{Use: "list"} var err error start := time.Now() stderr := captureStderr(t, func() { @@ -393,6 +508,45 @@ func TestSharedHoldAdvertisedToChildren(t *testing.T) { } } +// A strict readonly command on a never-gated workspace holds nothing, so it +// must not advertise a shared hold: a child trusting the marker would skip the +// writer-fairness queue for a maintenance operation this process does not +// block. Once the gate files exist the strict readonly hold is real, and it is +// advertised like any other. +func TestSharedHoldAdvertisedOnlyWhenAGateIsHeld(t *testing.T) { + resetGateTestEnv(t) + advertiseSharedHoldEnabled = true + t.Cleanup(func() { advertiseSharedHoldEnabled = false }) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + list := &cobra.Command{Use: "list"} + ctx := context.Background() + + setReadonlyMode(t, true) + if err := acquireCommandWorkspaceGates(ctx, list, beadsDir); err != nil { + t.Fatal(err) + } + if got := os.Getenv(workspacegate.InheritedHoldEnv); got != "" { + t.Fatalf("%s while a strict readonly command holds nothing = %q, want it left empty", workspacegate.InheritedHoldEnv, got) + } + releaseWorkspaceGates() + + // A normal command materializes the gate files. + setReadonlyMode(t, false) + if err := acquireCommandWorkspaceGates(ctx, list, beadsDir); err != nil { + t.Fatal(err) + } + releaseWorkspaceGates() + + setReadonlyMode(t, true) + if err := acquireCommandWorkspaceGates(ctx, list, beadsDir); err != nil { + t.Fatal(err) + } + if got, want := os.Getenv(workspacegate.InheritedHoldEnv), strconv.Itoa(os.Getpid()); got != want { + t.Fatalf("%s while a strict readonly command holds existing gates = %q, want %q", workspacegate.InheritedHoldEnv, got, want) + } +} + func TestSharedGateWaitEnv(t *testing.T) { for _, tc := range []struct { raw, hook string @@ -806,3 +960,180 @@ func TestEnvWithoutSharedHoldMarker(t *testing.T) { t.Fatal("envWithoutSharedHoldMarker dropped unrelated variables") } } + +// setReadonlyMode pins strict --readonly for one test, as PersistentPreRunE +// leaves readonlyMode by the time it acquires the gates. +func setReadonlyMode(t *testing.T, on bool) { + t.Helper() + original := readonlyMode + readonlyMode = on + t.Cleanup(func() { readonlyMode = original }) +} + +// newGateTestServerWorkspace is newGateTestWorkspace for a local-server +// workspace: its physical root is .beads/dolt, not .beads/embeddeddolt. +func newGateTestServerWorkspace(t *testing.T) string { + t.Helper() + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o755); err != nil { + t.Fatal(err) + } + meta := `{"backend":"dolt","database":"beads.db","dolt_mode":"server"}` + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(meta), 0o600); err != nil { + t.Fatal(err) + } + return beadsDir +} + +// dirEntryNames lists dir's immediate entries by name, so a test can prove a +// command left the directory exactly as it found it. +func dirEntryNames(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + return names +} + +// Strict --readonly is mutation-free (effectiveRootStorePolicy), so taking the +// command's gates must not create any gate file: not the workspace gate beside +// .beads, and not the physical-root gate inside it (.beads/embeddeddolt for an +// embedded workspace, .beads/dolt for a local server one). +func TestAcquireCommandWorkspaceGatesStrictReadonlyCreatesNoGateFiles(t *testing.T) { + layouts := []struct { + name string + build func(*testing.T) string + }{ + {"embedded", newGateTestWorkspace}, + {"local server", newGateTestServerWorkspace}, + } + for _, tc := range layouts { + t.Run(tc.name, func(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + setReadonlyMode(t, true) + beadsDir := tc.build(t) + projectDir := filepath.Dir(beadsDir) + projectBefore, beadsBefore := dirEntryNames(t, projectDir), dirEntryNames(t, beadsDir) + + list := &cobra.Command{Use: "list"} + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("strict readonly acquisition on a never-gated workspace: %v", err) + } + // SHARED fail-open also returns nil and creates nothing; it is + // told apart by running with no handle at all. + if workspaceGateHandle == nil { + t.Fatal("strict readonly acquisition on a never-gated workspace ran ungated, want a handle that holds nothing") + } + if !workspaceGateHandle.Empty() { + t.Fatal("strict readonly acquisition on a never-gated workspace locked a gate file") + } + if got := dirEntryNames(t, projectDir); !slices.Equal(got, projectBefore) { + t.Errorf("strict readonly changed %s\n before: %v\n after: %v", projectDir, projectBefore, got) + } + if got := dirEntryNames(t, beadsDir); !slices.Equal(got, beadsBefore) { + t.Errorf("strict readonly changed %s\n before: %v\n after: %v", beadsDir, beadsBefore, got) + } + }) + } +} + +// Not creating a gate file must not mean not honoring one. Once a workspace's +// gate files exist (any normal bd command creates them), a strict readonly +// command still holds them SHARED — so maintenance sees it and refuses to run +// over it — and is still refused while maintenance holds the gate. +func TestAcquireCommandWorkspaceGatesStrictReadonlyStillHonorsExistingGates(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + list := &cobra.Command{Use: "list"} + ctx := context.Background() + + // A normal command materializes the gate files. + setReadonlyMode(t, false) + if err := acquireCommandWorkspaceGates(ctx, list, beadsDir); err != nil { + t.Fatal(err) + } + releaseWorkspaceGates() + + setReadonlyMode(t, true) + if err := acquireCommandWorkspaceGates(ctx, list, beadsDir); err != nil { + t.Fatalf("strict readonly acquisition against existing gate files: %v", err) + } + gate, err := workspacegate.ForWorkspace(beadsDir) + if err != nil { + t.Fatal(err) + } + if _, err := gate.Acquire(ctx, workspacegate.Exclusive, workspacegate.Options{}); !errors.Is(err, workspacegate.ErrBusy) { + t.Fatalf("exclusive acquire while a strict readonly command is running: err = %v, want ErrBusy", err) + } + releaseWorkspaceGates() + + holder, err := gate.Acquire(ctx, workspacegate.Exclusive, workspacegate.Options{Reason: "test maintenance"}) + if err != nil { + t.Fatalf("exclusive acquire after the strict readonly command released: %v", err) + } + defer func() { _ = holder.Release() }() + // Bound the shared wait: resetGateTestEnv clears it, which means the 30s default. + t.Setenv(sharedGateWaitEnv, "300ms") + if err := acquireCommandWorkspaceGates(ctx, list, beadsDir); err == nil { + t.Fatal("strict readonly acquisition under a foreign exclusive holder must abort, got nil error") + } +} + +// Only STRICT readonly skips creating the gate files. A normal command keeps +// creating them: that is what makes it visible to a maintenance operation that +// starts later (the exclusive acquirer locks the same file), so the first gate +// holder on a workspace cannot go unseen. +func TestAcquireCommandWorkspaceGatesNormalCommandStillCreatesGateFiles(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + setReadonlyMode(t, false) + beadsDir := newGateTestWorkspace(t) + + list := &cobra.Command{Use: "list"} + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatal(err) + } + for _, gateFile := range []string{ + filepath.Join(filepath.Dir(beadsDir), ".beads.gate.lock"), + filepath.Join(beadsDir, "embeddeddolt.gate.lock"), + } { + if _, err := os.Stat(gateFile); err != nil { + t.Errorf("a normal command must create its gate file: %v", err) + } + } +} + +// End to end: the real `bd --readonly` leaves no gate file behind. The +// in-process tests above pin acquireCommandWorkspaceGates; this one pins that +// the command line reaches it with strict readonly already resolved. +func TestStrictReadonlyCommandCreatesNoGateFiles(t *testing.T) { + bdBin := buildBDForInitTests(t) + beadsDir := newGateTestServerWorkspace(t) + projectDir := filepath.Dir(beadsDir) + + cmd := exec.Command(bdBin, "--readonly", "list") + cmd.Dir = projectDir + cmd.Env = hermeticInitEnv(t.TempDir(), "BEADS_DIR="+beadsDir, "BEADS_DOLT_AUTO_START=0") + // No Dolt server sits behind this workspace, so the command is expected to + // fail when it opens the store; only what it left on disk matters here. + out, _ := cmd.CombinedOutput() + + var gateFiles []string + for _, dir := range []string{projectDir, beadsDir} { + for _, name := range dirEntryNames(t, dir) { + if strings.Contains(name, ".gate.lock") { + gateFiles = append(gateFiles, filepath.Join(dir, name)) + } + } + } + if len(gateFiles) != 0 { + t.Fatalf("bd --readonly created gate files %v\noutput: %s", gateFiles, out) + } +} diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel index 85a5fefdf..4928df002 100644 --- a/internal/workspacegate/BUILD.bazel +++ b/internal/workspacegate/BUILD.bazel @@ -10,7 +10,10 @@ go_library( go_test( name = "workspacegate_test", - srcs = ["gate_test.go"], + srcs = [ + "gate_test.go", + "readonly_wait_test.go", + ], embed = [":workspacegate"], deps = ["//internal/testutil/bazeltest"], ) diff --git a/internal/workspacegate/gate.go b/internal/workspacegate/gate.go index 754c14400..d0d27516c 100644 --- a/internal/workspacegate/gate.go +++ b/internal/workspacegate/gate.go @@ -112,6 +112,15 @@ type Options struct { // effort, from the advisory sidecar). AcquireAll fires it at most // once across the whole gate set. OnWait func(holder string) + // ReadOnly makes a SHARED acquisition free of side effects, for commands + // that promise not to mutate the workspace (strict --readonly). The gate + // file is opened O_RDONLY and never created: a gate file that does not + // exist means nothing has ever been gated here (the reading ExclusiveHolder + // uses), so Acquire succeeds holding nothing. An existing file is held + // SHARED as usual, so maintenance still sees the holder and a live + // exclusive holder still turns it away. Ignored for Exclusive: a + // maintenance operation has to be able to create the file it locks. + ReadOnly bool // IgnoreQueuedExclusive makes a waiting Shared acquisition skip the // writer-fairness check. Set it only when an ANCESTOR process already // holds this gate shared (see InheritedSharedHold): the queued @@ -372,6 +381,11 @@ type Handle struct { // Mode reports how this handle was acquired. func (h *Handle) Mode() Mode { return h.mode } +// Empty reports whether the acquisition locked nothing: a ReadOnly shared +// acquisition of a gate file that does not exist (see Options.ReadOnly). An +// empty handle blocks no acquirer. Release does not change the answer. +func (h *Handle) Empty() bool { return h == nil || h.f == nil } + // Release drops the lock and closes the file handle. For exclusive // handles it also best-effort removes the holder-info sidecar. The gate // file itself is intentionally never removed: deleting a lock file that @@ -382,6 +396,11 @@ func (h *Handle) Release() error { return nil } h.once.Do(func() { + if h.f == nil { + // A ReadOnly acquisition of a gate file that does not exist: + // nothing was locked, so there is nothing to drop. + return + } var errs []error if h.mode == Exclusive { // Remove the sidecar BEFORE unlocking: after the unlock a new @@ -455,6 +474,9 @@ func (h *Handle) Release() error { // Lock order is unchanged in substance: the intent lock of a gate is taken // only while acquiring that gate, so it slots directly before the gate in // AcquireAll's sorted total order and cannot form a cycle. +// +// A Shared acquisition with Options.ReadOnly never creates the gate file: +// see that option for what it holds instead. func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, error) { if g.path == "" { return nil, errors.New("workspacegate: zero Gate; use ForWorkspace/ForPhysicalRoot") @@ -475,8 +497,20 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er } deadline := time.Now().Add(opts.Wait) - f, err := os.OpenFile(g.path, os.O_CREATE|os.O_RDWR, 0o600) + // An exclusive acquirer always has to create the file it locks, so + // ReadOnly applies to shared acquisition only. + readOnly := opts.ReadOnly && mode == Shared + flag := os.O_CREATE | os.O_RDWR + if readOnly { + flag = os.O_RDONLY + } + f, err := os.OpenFile(g.path, flag, 0o600) if err != nil { + if readOnly && os.IsNotExist(err) { + // No gate file: nothing has ever gated here, so there is no + // holder to see and nothing to hold. + return &Handle{gate: g, mode: mode}, nil + } return nil, fmt.Errorf("workspacegate: open gate %s: %w", g.path, err) } @@ -705,6 +739,21 @@ type MultiHandle struct { handles []*Handle } +// Empty reports whether the acquisition locked no gate at all: every gate in +// the set was a ReadOnly acquisition of a gate file that does not exist (see +// Handle.Empty). +func (m *MultiHandle) Empty() bool { + if m == nil { + return true + } + for _, h := range m.handles { + if !h.Empty() { + return false + } + } + return true +} + // Release drops every held gate in reverse order. Idempotent. func (m *MultiHandle) Release() error { if m == nil { diff --git a/internal/workspacegate/gate_test.go b/internal/workspacegate/gate_test.go index 391a6d50d..eb5ddd100 100644 --- a/internal/workspacegate/gate_test.go +++ b/internal/workspacegate/gate_test.go @@ -322,6 +322,184 @@ func TestExclusiveHolderDoesNotDisturbSharedAcquirers(t *testing.T) { <-done } +// A ReadOnly shared acquisition on a workspace that has never been gated must +// succeed and create nothing: an absent gate file means nothing has ever been +// gated here (the reading ExclusiveHolder already uses), so there is no holder +// to observe and nothing to hold. This is what lets a strict --readonly +// command stay mutation-free. +func TestReadOnlySharedAcquireCreatesNothing(t *testing.T) { + g, dir := testGate(t) + + h, err := g.Acquire(context.Background(), Shared, Options{ReadOnly: true}) + if err != nil { + t.Fatalf("ReadOnly shared Acquire on a never-gated workspace: %v, want success (nothing to hold)", err) + } + for i := 1; i <= 2; i++ { + if err := h.Release(); err != nil { + t.Fatalf("Release #%d of the ReadOnly handle: %v", i, err) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var created []string + for _, e := range entries { + created = append(created, e.Name()) + } + if len(created) != 0 { + t.Fatalf("ReadOnly Acquire created %v in %s, want nothing", created, dir) + } +} + +// Against a gate file that already exists, a ReadOnly shared acquisition is a +// real shared hold — it coexists with other shared holders, is refused by a +// live exclusive holder, and keeps an exclusive acquirer out until released. +// Dropping the create must not make strict readonly commands invisible to +// maintenance. +func TestReadOnlySharedAcquireHoldsExistingGate(t *testing.T) { + g, _ := testGate(t) + // Materialize the gate file once, as any workspace that has run bd has. + h0 := mustAcquire(t, g, Shared, Options{}) + _ = h0.Release() + + h := mustAcquire(t, g, Shared, Options{ReadOnly: true}) + other := mustAcquire(t, g, Shared, Options{}) + _ = other.Release() + if _, err := g.Acquire(context.Background(), Exclusive, Options{}); !errors.Is(err, ErrBusy) { + t.Fatalf("exclusive under a ReadOnly shared holder: err = %v, want ErrBusy", err) + } + if err := h.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + + hEx := mustAcquire(t, g, Exclusive, Options{Reason: "after the ReadOnly holder left"}) + defer hEx.Release() + if _, err := g.Acquire(context.Background(), Shared, Options{ReadOnly: true}); !errors.Is(err, ErrBusy) { + t.Fatalf("ReadOnly shared under an exclusive holder: err = %v, want ErrBusy", err) + } +} + +// The gate file is opened O_RDONLY, not merely without O_CREATE: a strict +// readonly command in a sandbox whose gate file it cannot write must still be +// able to hold it (flock does not need a writable descriptor). +func TestReadOnlySharedAcquireNeedsNoWriteAccess(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits do not gate opens on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root ignores file permission bits") + } + g, _ := testGate(t) + h0 := mustAcquire(t, g, Shared, Options{}) + _ = h0.Release() + if err := os.Chmod(g.Path(), 0o400); err != nil { + t.Fatal(err) + } + + h, err := g.Acquire(context.Background(), Shared, Options{ReadOnly: true}) + if err != nil { + t.Fatalf("ReadOnly shared Acquire on a read-only gate file: %v, want success", err) + } + _ = h.Release() +} + +// ReadOnly applies to shared acquisition only: an exclusive acquirer has to be +// able to create the gate file it locks, so the option cannot suppress that. +func TestReadOnlyOptionIgnoredForExclusive(t *testing.T) { + g, _ := testGate(t) + + h, err := g.Acquire(context.Background(), Exclusive, Options{ReadOnly: true, Reason: "readonly is ignored"}) + if err != nil { + t.Fatalf("exclusive Acquire with ReadOnly set: %v", err) + } + defer h.Release() + if _, err := os.Stat(g.Path()); err != nil { + t.Fatalf("exclusive Acquire must create the gate file it locks: %v", err) + } + if _, err := g.Acquire(context.Background(), Shared, Options{}); !errors.Is(err, ErrBusy) { + t.Fatalf("shared under that exclusive holder: err = %v, want ErrBusy", err) + } +} + +// AcquireAll passes the option to every gate: gates whose file exists are held +// shared, gates whose file does not are skipped and stay absent — for the +// workspace gate beside .beads and the physical-root gate inside it alike. +func TestAcquireAllReadOnlySkipsAbsentGates(t *testing.T) { + dir := t.TempDir() + beadsDir := filepath.Join(dir, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + ws, err := ForWorkspace(beadsDir) + if err != nil { + t.Fatal(err) + } + root, err := ForPhysicalRoot(filepath.Join(beadsDir, "embeddeddolt")) + if err != nil { + t.Fatal(err) + } + // Only the workspace gate has ever been used. + h0 := mustAcquire(t, ws, Shared, Options{}) + _ = h0.Release() + + m, err := AcquireAll(context.Background(), Shared, Options{ReadOnly: true}, ws, root) + if err != nil { + t.Fatalf("AcquireAll with ReadOnly over an existing and an absent gate: %v", err) + } + if _, err := ws.Acquire(context.Background(), Exclusive, Options{}); !errors.Is(err, ErrBusy) { + t.Fatalf("exclusive on the existing workspace gate under AcquireAll: err = %v, want ErrBusy", err) + } + if err := m.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + if _, err := os.Lstat(root.Path()); !os.IsNotExist(err) { + t.Fatalf("AcquireAll with ReadOnly created the absent physical-root gate %s (stat err: %v)", root.Path(), err) + } +} + +// Empty separates a ReadOnly acquisition that locked nothing from a real +// hold: callers use it to decide whether there is a hold to tell anyone about. +// One gate file that exists makes the whole set a real hold. +func TestEmptyReportsWhetherAnythingIsLocked(t *testing.T) { + never, _ := testGate(t) // never gated: its file stays absent + used, _ := testGate(t) // gated once, below + readOnly := Options{ReadOnly: true} + + if !(*Handle)(nil).Empty() || !(*MultiHandle)(nil).Empty() { + t.Fatal("a nil handle must report Empty") + } + absent, err := AcquireAll(context.Background(), Shared, readOnly, never, used) + if err != nil { + t.Fatalf("ReadOnly AcquireAll over two absent gates: %v", err) + } + if !absent.Empty() { + t.Fatal("ReadOnly AcquireAll over two absent gates reported a hold, want Empty") + } + _ = absent.Release() + + h0 := mustAcquire(t, used, Shared, Options{}) + if h0.Empty() { + t.Fatal("a plain shared acquisition reported Empty") + } + _ = h0.Release() + if h0.Empty() { + t.Fatal("Release changed Empty; it reports the acquisition, not the current state") + } + + partial, err := AcquireAll(context.Background(), Shared, readOnly, never, used) + if err != nil { + t.Fatalf("ReadOnly AcquireAll over an absent and an existing gate: %v", err) + } + defer partial.Release() + if partial.Empty() { + t.Fatal("ReadOnly AcquireAll holding the existing gate reported Empty") + } + if _, err := os.Lstat(never.Path()); !os.IsNotExist(err) { + t.Fatalf("the absent gate %s must stay absent (stat err: %v)", never.Path(), err) + } +} + // A cancelled context must surface as the context error even in // fail-fast mode (Wait == 0), so callers can tell shutdown from // contention. diff --git a/internal/workspacegate/readonly_wait_test.go b/internal/workspacegate/readonly_wait_test.go new file mode 100644 index 000000000..18b623d06 --- /dev/null +++ b/internal/workspacegate/readonly_wait_test.go @@ -0,0 +1,183 @@ +package workspacegate + +import ( + "context" + "errors" + "os" + "slices" + "testing" + "time" +) + +// A strict-readonly SHARED acquisition (Options.ReadOnly) and a waiting one +// (Options.Wait, with its writer-fairness intent queue) meet in exactly one +// place: a ReadOnly acquisition with Wait > 0. The ReadOnly tests in +// gate_test.go are all wait-less, so the tests below pin that seam: waiting +// must not make a ReadOnly acquisition create anything (no gate file, no +// intent file, no holder-info sidecar), and ReadOnly must not change how long +// a shared acquisition waits for an exclusive holder. +// +// None of them touches maxIntentHold or any other package variable a test +// shortens, and none runs in parallel with anything. + +// gateDirEntries lists dir's entries by name (os.ReadDir sorts them), so a +// test can prove an acquisition left the gate's directory as it found it. +func gateDirEntries(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + return names +} + +// (i) The gate file exists and no intent file does: a waiting ReadOnly shared +// acquisition takes the gate on its first attempt (OnWait never fires) and +// creates nothing beside it: no .intent, no .intent.info, no .info. +func TestReadOnlyWaitingSharedAcquireOfExistingGateCreatesNothing(t *testing.T) { + g, dir := testGate(t) + // Materialize the gate file the way any ordinary command does. + _ = mustAcquire(t, g, Shared, Options{}).Release() + before := gateDirEntries(t, dir) + if _, err := os.Stat(g.Path()); err != nil { + t.Fatalf("precondition: the gate file must exist: %v", err) + } + if _, err := os.Stat(g.intentPath()); !os.IsNotExist(err) { + t.Fatalf("precondition: no intent file may exist, stat err = %v", err) + } + + const wait = time.Second + var waited []string + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + ReadOnly: true, + Wait: wait, + OnWait: func(holder string) { waited = append(waited, holder) }, + }) + elapsed := time.Since(start) + if err != nil { + t.Fatalf("waiting ReadOnly shared Acquire of a free, existing gate: %v", err) + } + defer func() { _ = h.Release() }() + if len(waited) != 0 { + t.Errorf("the first attempt did not take the free gate: OnWait fired with %q", waited) + } + if elapsed >= wait { + t.Errorf("took %s, want far under its %s budget: nothing was holding the gate", elapsed, wait) + } + if h.f == nil { + t.Fatal("handle holds no file: against an existing gate file a ReadOnly acquisition is a real shared hold") + } + if got := gateDirEntries(t, dir); !slices.Equal(got, before) { + t.Errorf("waiting ReadOnly acquisition changed %s\n before: %v\n after: %v", dir, before, got) + } + // It is a real shared hold: a maintenance operation is turned away. + if _, err := g.Acquire(context.Background(), Exclusive, Options{}); !errors.Is(err, ErrBusy) { + t.Fatalf("exclusive Acquire under the ReadOnly shared holder: err = %v, want ErrBusy", err) + } + if err := h.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + if got := gateDirEntries(t, dir); !slices.Equal(got, before) { + t.Errorf("after release %s\n before: %v\n after: %v", dir, before, got) + } +} + +// (ii) No gate file: nothing has ever gated here. A waiting ReadOnly shared +// acquisition returns at once with an empty handle (it holds no file and has +// nothing to wait for) and leaves the directory exactly as it was: it neither +// creates the gate file nor waits for one to appear. +func TestReadOnlyWaitingSharedAcquireOfMissingGateHoldsNothing(t *testing.T) { + g, dir := testGate(t) + before := gateDirEntries(t, dir) + if len(before) != 0 { + t.Fatalf("precondition: %s must start empty, has %v", dir, before) + } + + const wait = time.Second + var waited []string + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + ReadOnly: true, + Wait: wait, + OnWait: func(holder string) { waited = append(waited, holder) }, + }) + elapsed := time.Since(start) + if err != nil { + t.Fatalf("waiting ReadOnly shared Acquire with no gate file: %v, want success holding nothing", err) + } + if h == nil || h.f != nil { + t.Fatalf("handle = %+v, want an empty handle (no gate file was opened)", h) + } + if h.Mode() != Shared { + t.Errorf("handle mode = %s, want %s", h.Mode(), Shared) + } + if len(waited) != 0 { + t.Errorf("OnWait fired with %q: with no gate file there is nothing to wait for", waited) + } + if elapsed >= wait { + t.Errorf("took %s, want far under its %s budget: a missing gate file is answered before any retry", elapsed, wait) + } + if got := gateDirEntries(t, dir); !slices.Equal(got, before) { + t.Errorf("waiting ReadOnly acquisition changed %s\n before: %v\n after: %v", dir, before, got) + } + for i := 1; i <= 2; i++ { + if err := h.Release(); err != nil { + t.Fatalf("Release #%d of the empty handle: %v", i, err) + } + } + if got := gateDirEntries(t, dir); !slices.Equal(got, before) { + t.Errorf("after release %s\n before: %v\n after: %v", dir, before, got) + } +} + +// (iii) The gate file exists and is held EXCLUSIVELY: a waiting ReadOnly +// shared acquisition waits out its own budget, exactly like any shared +// acquisition, and then fails with ErrBusy. ReadOnly is not special-cased: +// it neither fails at once nor waits longer (the shared default is 30s, the +// init budget 60s). It also creates nothing while it waits, which is where +// the fairness probe (ExclusiveQueued) meets ReadOnly: the probe opens the +// intent file read-only and treats a missing one as "nothing queued". +func TestReadOnlyWaitingSharedAcquireUnderExclusiveHolderWaitsItsBudget(t *testing.T) { + g, dir := testGate(t) + holder := mustAcquire(t, g, Exclusive, Options{Reason: "test maintenance"}) + defer func() { _ = holder.Release() }() + before := gateDirEntries(t, dir) + + const wait = 300 * time.Millisecond + waited := 0 + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + ReadOnly: true, + Wait: wait, + OnWait: func(string) { waited++ }, + }) + elapsed := time.Since(start) + if !errors.Is(err, ErrBusy) { + if h != nil { + _ = h.Release() + } + t.Fatalf("waiting ReadOnly shared Acquire under an exclusive holder: err = %v, want ErrBusy", err) + } + // Acquire's deadline is taken after start, and it reports ErrBusy only + // once that deadline has passed, so this lower bound is exact. + if elapsed < wait { + t.Errorf("gave up after %s, before its %s budget ran out: ReadOnly must wait like any shared acquisition", elapsed, wait) + } + // The loop overshoots by at most one poll interval (100ms by default). + // The ceiling is far more generous than that so a loaded host cannot make + // this flaky; it still catches any special-cased budget. + if limit := wait + 2*time.Second; elapsed > limit { + t.Errorf("gave up after %s, want about its %s budget (at most %s)", elapsed, wait, limit) + } + if waited != 1 { + t.Errorf("OnWait fired %d times, want once (after the first busy attempt)", waited) + } + if got := gateDirEntries(t, dir); !slices.Equal(got, before) { + t.Errorf("waiting ReadOnly acquisition changed %s\n before: %v\n after: %v", dir, before, got) + } +} From 3a5ade3a76e2d6c70395b93255ce3914d1200c39 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 04:51:18 +0000 Subject: [PATCH 22/54] test(cmd/bd): pin workspace discovery refusal text and exit codes (#7677) * test(cmd/bd): pin workspace discovery refusal text and exit codes Add before-parity pins for what an ordinary bd prints, and with which exit code, when workspace discovery cannot select a usable workspace: - no .beads, an empty .beads, or a .beads with config.yaml but no metadata.json and no database ("no beads database found", and bd where's text and --json refusals); - a corrupt metadata.json, for list, create, where (all refuse) and version (warns, exits 0); - an exported BEADS_DIR naming a missing path, an empty directory, a regular file or a corrupt workspace, run from a corrupt workspace so a fallback to cwd discovery would show; - .beads/.env: BEADS_DIR routing, an exported BEADS_DIR winning, BEADS_DB routing bd where but not a data command, BD_JSON applying only in the selected workspace, and a .env BD_BACKEND not being refused; - .beads/redirect: missing and non-workspace targets (warning, then the source), a corrupt target, a chain, and a redirect behind BEADS_DIR; - --db paths and names, and BEADS_DB / BD_DB, including the two inconsistencies main has today (a data command ignores BD_DB, and lets a discoverable cwd workspace shadow BEADS_DB, while bd where honors both); - BD_BACKEND / BD_DATABASE_BACKEND being refused before any of the above, for store-less commands too. The cases use workspaces with a corrupt metadata.json as sentinels: the refusal names the metadata.json it failed to parse, which shows the directory discovery selected without opening a store, so the store-less pins need neither Dolt nor cgo. One embedded-Dolt pin covers the store-backed .env case: BD_READONLY=true in .beads/.env refuses a write with the read-only text and exit 1, reads still run, and an exported BD_READONLY wins over the file. These pin today's origin/main, not an ideal. A change that reorders admission, discovery or checkBlockedEnvVars in the root PersistentPreRunE now has to change an expectation here and is reviewed as a test diff. Built from the merge base with the graph fork (46aa59cc65), bd passes every store-less pin; the fork tip fails two (a BEADS_DIR naming a regular file exits 5 with graph_not_initialized instead of exit 1, also when BD_BACKEND is set). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(cmd/bd): keep the discovery pins independent of PATH and cgo The version case under TestDiscoveryRefusalPinsCorruptMetadata pinned bd version's full stderr. bd version also appends a "Warning: multiple 'bd' binaries found in PATH" block when the caller's PATH holds more than one distinct bd binary (a release install plus a dev build, or Homebrew plus go install), and the fixture passes the caller's PATH through. Such a machine failed the pin spuriously. normalize now drops that block from stderr, as it already drops the sandbox auto-detection notice, so the rest of the case is still pinned exactly. The file also said the store-less pins need no cgo. They need a cgo bd. A CGO_ENABLED=0 bd is server-only, and its no-database hints add "run 'bd doctor' to diagnose", so a CGO_ENABLED=0 go test of the package failed 16 cases on the hint wording alone. The file now builds only with cgo, and its header says why. CI runs none of these in its CGO_ENABLED=0 lane, whose -run subset is fixed by PURE_CMD_BD_TESTS. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(cmd/bd): pin a working-directory .beads that is a regular file The ExplicitBeadsDir pins cover a BEADS_DIR that names a regular file. This adds the same case reached by the working-directory walk instead: a project whose .beads is a regular file. bd list and bd where both exit 1 with "Error: loading config: reading config: open $ROOT/p/.beads/metadata.json: not a directory". bd built from the fork merge base 46aa59cc65 passes both cases. The fork tip b9eebc31f8 fails both: it exits 5 with "graph_not_initialized: open $ROOT/p/.beads/graph-preview-format: not a directory", the same reorder the BEADS_DIR pin already catches. A .beads that is an unreadable directory (chmod 000) splits main and the fork the same way, but it is not pinned: a test run as root can still read the directory. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(cmd/bd): close the embedded discovery pin's environment Maintainer fixup from the adoption review of #7677. TestEmbeddedDiscoveryPinsEnvFileReadonly compares bd's output exactly, but its run helper started from bdEnv, which drops only BEADS_* and BD_ROUTING_MODE. Every other BD_* the caller exported reached bd: with BD_OTEL_METRICS_URL/BD_OTEL_LOGS_URL set (as on agent hosts) the deprecation warning lands in front of the read-only refusal, with BD_JSON_ENVELOPE=1 the empty list is printed as an envelope, and in a sandbox the "Sandbox detected" notice precedes the refusal; each failed the pin. The run helper now uses the store-less pins' closed environment, factored out of discoveryPinFixture.baseEnv as discoveryPinEnv (HOME stays the directory bd init used), and drops the sandbox notice from stderr with the same discoveryPinSandboxNotice pattern normalize now uses. normalize collapsed every consecutive repeat of a stderr line, which also hid a doubled "Error:" line. It now folds only repeated "Warning: redirect" lines, the ones discovery prints once per re-resolve. blocked-backend-not-checked used a corrupt-metadata sentinel, whose bind-time refusal comes before the legacy guard, so a blocked-variable check added after .env loading would not have changed its output. It now uses a config.yaml-only workspace and pins discovery's no-database refusal. Measured with the PR head's bd and test binaries under env -i: - the embedded pin passes on a clean environment and failed with BD_OTEL_*, BD_JSON_ENVELOPE=1, or kill(2) returning EPERM (strace inject); with this change it passes in all of them, plus an exported XDG_CONFIG_HOME, an existing BD_MIGRATION_FREEZE_FILE and the agent host's BD_*/BEADS_* set; - it still fails when .beads/.env overrides the caller (gotenv.OverLoad) or is not loaded; - the store-less suite passes 54/54 subtests; a bd that prints the legacy-guard error twice now fails 12 of them, and a bd that checks the blocked variables again after the legacy guard now fails blocked-backend-not-checked; both passed the suite before. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- .github/scripts/embedded-cmd-test-shards.txt | 1 + cmd/bd/BUILD.bazel | 2 + .../discovery_refusal_pins_embedded_test.go | 86 ++ cmd/bd/discovery_refusal_pins_test.go | 885 ++++++++++++++++++ 4 files changed, 974 insertions(+) create mode 100644 cmd/bd/discovery_refusal_pins_embedded_test.go create mode 100644 cmd/bd/discovery_refusal_pins_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index e05f241a2..95d0ef0f0 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -134,6 +134,7 @@ 100 53 TestEmbeddedIfRevisionOrdinaryCAS 100 53 TestEmbeddedInitSelectedExcludeRoutingStealthDecoyQuiet +100 54 TestEmbeddedDiscoveryPinsEnvFileReadonly 100 54 TestEmbeddedFormatJSONAliasMatchesJSON 100 54 TestEmbeddedSearchConcurrent diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f16e65cbb..3ac61dc7c 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -714,6 +714,8 @@ go_test( "dependency_hierarchy_embedded_test.go", "diff_embedded_test.go", "directory_flag_test.go", + "discovery_refusal_pins_embedded_test.go", + "discovery_refusal_pins_test.go", "doctor_backend_guard_test.go", "doctor_check_gate_test.go", "doctor_context_test.go", diff --git a/cmd/bd/discovery_refusal_pins_embedded_test.go b/cmd/bd/discovery_refusal_pins_embedded_test.go new file mode 100644 index 000000000..5bfda4176 --- /dev/null +++ b/cmd/bd/discovery_refusal_pins_embedded_test.go @@ -0,0 +1,86 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// TestEmbeddedDiscoveryPinsEnvFileReadonly is the store-backed member of the +// before-parity discovery pins (discovery_refusal_pins_test.go holds the +// store-less ones). It pins how an ordinary workspace applies a BD_READONLY +// supplied by its own .beads/.env: a write is refused with the read-only text +// and exit 1, also under --json, while reads still run; and a BD_READONLY the +// caller exports wins over the file. +func TestEmbeddedDiscoveryPinsEnvFileReadonly(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "pin") + if err := os.WriteFile(filepath.Join(beadsDir, ".env"), []byte("BD_READONLY=true\n"), 0o600); err != nil { + t.Fatalf("write .beads/.env: %v", err) + } + + run := func(extraEnv []string, args ...string) (exit int, stdout, stderr string) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + // The store-less pins' closed environment: no BD_*/BEADS_* the caller + // exports (BD_READONLY, BD_JSON_ENVELOPE, BD_OTEL_*, ...) reaches bd. + // HOME stays dir, where bd init ran. + cmd.Env = append(discoveryPinEnv(dir, filepath.Join(dir, "no-such-freeze-marker")), + "BEADS_DOLT_AUTO_START=0", "BEADS_NO_DAEMON=1") + cmd.Env = append(cmd.Env, extraEnv...) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + if err := cmd.Run(); err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("run bd %v: %v", args, err) + } + exit = exitErr.ExitCode() + } + return exit, out.String(), discoveryPinSandboxNotice.ReplaceAllLiteralString(errOut.String(), "") + } + + exit, stdout, stderr := run(nil, "create", "refused by .env", "--json") + if exit != 1 || stdout != "" || stderr != "Error: operation 'create' is not allowed in read-only mode\n" { + t.Fatalf("create under .env BD_READONLY=true: exit=%d stdout=%q stderr=%q; want exit=1, no stdout, the read-only refusal", exit, stdout, stderr) + } + + exit, stdout, stderr = run(nil, "list", "--json") + if exit != 0 || stdout != "[]\n" { + t.Fatalf("list under .env BD_READONLY=true: exit=%d stdout=%q stderr=%q; want exit=0 and an empty list", exit, stdout, stderr) + } + + exit, stdout, stderr = run([]string{"BD_READONLY=false"}, "create", "allowed by the caller", "--json") + if exit != 0 { + t.Fatalf("create with exported BD_READONLY=false: exit=%d stdout=%q stderr=%q; want the caller's setting to win", exit, stdout, stderr) + } + var created struct { + Title string `json:"title"` + } + if err := json.Unmarshal([]byte(stdout), &created); err != nil || created.Title != "allowed by the caller" { + t.Fatalf("create with exported BD_READONLY=false printed %q (decode error %v)", stdout, err) + } + + exit, stdout, stderr = run(nil, "list", "--json") + var listed []struct { + Title string `json:"title"` + } + if exit != 0 { + t.Fatalf("list after the allowed create: exit=%d stderr=%q", exit, stderr) + } + if err := json.Unmarshal([]byte(stdout), &listed); err != nil || len(listed) != 1 || listed[0].Title != "allowed by the caller" { + t.Fatalf("list after the allowed create = %q (decode error %v); want only the allowed issue, the refused create wrote nothing", stdout, err) + } +} diff --git a/cmd/bd/discovery_refusal_pins_test.go b/cmd/bd/discovery_refusal_pins_test.go new file mode 100644 index 000000000..c0d4254f9 --- /dev/null +++ b/cmd/bd/discovery_refusal_pins_test.go @@ -0,0 +1,885 @@ +//go:build cgo + +package main + +// Before-parity pins for workspace discovery refusals. +// +// These tests hold the exact text and exit code an ordinary bd prints when +// workspace discovery cannot select a usable workspace: a missing or corrupt +// metadata.json, an explicit BEADS_DIR that names no workspace, selection +// supplied by .beads/.env, .beads/redirect files, --db / BEADS_DB / BD_DB +// selection, and the blocked backend environment variables. They pin what +// origin/main does today, not what it ideally should do. A change that +// reorders admission, discovery or checkBlockedEnvVars in the root +// PersistentPreRunE has to change an expectation here, so it shows up in +// review as a test diff rather than as an unexplained behavior change. +// +// Every case runs the real bd binary in a fresh fixture and stops before any +// store is opened, so these need no Dolt server or database. Fixture +// workspaces with a corrupt metadata.json double as sentinels: the refusal +// names the metadata.json it failed to parse, so the text shows which +// directory discovery selected without opening a database. +// +// The pinned no-database hints are an embedded-capable bd's. A CGO_ENABLED=0 +// bd is server-only and adds "run 'bd doctor' to diagnose" to them, so this +// file builds only with cgo, and the bd it runs (built by +// buildBDForInitTests, or BEADS_TEST_BD_BINARY) must be a cgo build too. + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// pinNoDatabaseRefusal is what a store-requiring command prints when +// discovery finds no workspace with a database. +const pinNoDatabaseRefusal = "Error: no beads database found\n" + + "Hint: run 'bd where' to inspect the resolved workspace, or 'bd init' to create a new database\n" + + " or set BEADS_DIR to point to your .beads directory\n" + +// pinNoWorkspaceWhereRefusal is bd where's text refusal when no workspace is +// selected. +const pinNoWorkspaceWhereRefusal = "Error: No active beads workspace found.\n" + + "Hint: check BEADS_DIR/worktree setup, or run 'bd init' to create a new database\n" + +// pinNoWorkspaceWhereJSON is bd where --json's refusal. It goes to stdout. +const pinNoWorkspaceWhereJSON = `{ + "error": "no_beads_directory", + "hint": "check BEADS_DIR/worktree setup, or run 'bd init' to create a new database", + "message": "No active beads workspace found.", + "schema_version": 1 +} +` + +// pinCorruptMetadata is the content of every corrupt metadata.json fixture. +const pinCorruptMetadata = "{\n" + +// pinCorruptMetadataWarning is the warning printed while the selected +// workspace is bound, before the refusal itself. +func pinCorruptMetadataWarning(beadsDir string) string { + return "warning: load " + beadsDir + "/metadata.json: parsing config: unexpected end of JSON input; " + + "no storage database was opened or modified (storage mode unknown; data commands refuse to fall back to the embedded store)\n" +} + +// pinCorruptMetadataRefusal is the full stderr of a command refused because +// the selected workspace's metadata.json does not parse. +func pinCorruptMetadataRefusal(beadsDir string) string { + return pinCorruptMetadataWarning(beadsDir) + + "Error: loading config: parsing config: unexpected end of JSON input\n" +} + +// pinBlockedEnvRefusal is checkBlockedEnvVars' refusal for name. +func pinBlockedEnvRefusal(name string) string { + return "Error: " + name + " env var is not supported and has been removed to prevent data fragmentation.\n" + + "Unset " + name + "; storage selection comes from .beads/metadata.json. " + + "To choose a different supported backend, follow 'bd help init-safety'; do not edit metadata.json by hand\n" +} + +// discoveryPinCase is one bd invocation and its exact expected result. The +// arguments and the expected strings write the fixture root as $ROOT. +type discoveryPinCase struct { + name string + // setup builds the fixture and returns the working directory and any + // environment the invocation adds to the hermetic base. + setup func(f *discoveryPinFixture) (cwd string, env []string) + args []string + exit int + stdout string + stderr string + // stderrPrefix compares only the start of stderr. Use it only where the + // rest of the output is not part of the pin. + stderrPrefix bool +} + +func runDiscoveryPinCases(t *testing.T, cases []discoveryPinCase) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("the pinned texts embed POSIX paths") + } + t.Parallel() + bd := buildBDForInitTests(t) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := newDiscoveryPinFixture(t) + cwd, env := tc.setup(f) + args := make([]string, len(tc.args)) + for i, arg := range tc.args { + args[i] = strings.ReplaceAll(arg, "$ROOT", f.root) + } + exit, stdout, stderr := f.run(bd, cwd, env, args...) + if exit != tc.exit { + t.Errorf("bd %s: exit = %d, want %d", strings.Join(tc.args, " "), exit, tc.exit) + } + if stdout != tc.stdout { + t.Errorf("bd %s: stdout differs\n got: %q\nwant: %q", strings.Join(tc.args, " "), stdout, tc.stdout) + } + gotStderr := stderr + if tc.stderrPrefix && len(gotStderr) > len(tc.stderr) { + gotStderr = gotStderr[:len(tc.stderr)] + } + if gotStderr != tc.stderr { + t.Errorf("bd %s: stderr differs\n got: %q\nwant: %q", strings.Join(tc.args, " "), stderr, tc.stderr) + } + }) + } +} + +type discoveryPinFixture struct { + t *testing.T + root string + forms []string // spellings of root to rewrite as $ROOT, longest first +} + +func newDiscoveryPinFixture(t *testing.T) *discoveryPinFixture { + t.Helper() + root := t.TempDir() + forms := []string{root} + if resolved, err := filepath.EvalSymlinks(root); err == nil && resolved != root { + forms = append(forms, resolved) + } + sort.Slice(forms, func(i, j int) bool { return len(forms[i]) > len(forms[j]) }) + f := &discoveryPinFixture{t: t, root: root, forms: forms} + f.mkdir("home") + return f +} + +// path returns the absolute path of a root-relative slash path. +func (f *discoveryPinFixture) path(rel string) string { + return filepath.Join(f.root, filepath.FromSlash(rel)) +} + +func (f *discoveryPinFixture) mkdir(rel string) string { + f.t.Helper() + dir := f.path(rel) + if err := os.MkdirAll(dir, 0o700); err != nil { + f.t.Fatalf("mkdir %s: %v", dir, err) + } + return dir +} + +func (f *discoveryPinFixture) write(rel, content string) { + f.t.Helper() + if err := os.WriteFile(f.path(rel), []byte(content), 0o600); err != nil { + f.t.Fatalf("write %s: %v", rel, err) + } +} + +// project creates rel as a git repository with no .beads. The repository +// root is the discovery walk's boundary, so nothing above the fixture leaks +// into a case. +func (f *discoveryPinFixture) project(rel string) string { + f.t.Helper() + dir := f.mkdir(rel) + initGitRepoAt(f.t, dir) + return dir +} + +// workspace creates project rel with a .beads directory holding files, each +// named relative to .beads. +func (f *discoveryPinFixture) workspace(rel string, files map[string]string) string { + f.t.Helper() + dir := f.project(rel) + f.mkdir(rel + "/.beads") + for name, content := range files { + f.write(rel+"/.beads/"+name, content) + } + return dir +} + +// corruptWorkspace creates project rel whose metadata.json does not parse, +// plus any extra .beads files. +func (f *discoveryPinFixture) corruptWorkspace(rel string, extra map[string]string) string { + f.t.Helper() + files := map[string]string{"metadata.json": pinCorruptMetadata} + for name, content := range extra { + files[name] = content + } + return f.workspace(rel, files) +} + +// yamlWorkspace creates project rel whose .beads holds only a config.yaml +// (no metadata.json and no database), plus any extra .beads files. +func (f *discoveryPinFixture) yamlWorkspace(rel string, extra map[string]string) string { + f.t.Helper() + files := map[string]string{"config.yaml": "# discovery pin fixture\n"} + for name, content := range extra { + files[name] = content + } + return f.workspace(rel, files) +} + +// discoveryPinEnv is the caller environment without any BEADS_*/BD_* +// selector or setting and without git repository-location variables, with +// HOME and XDG_CONFIG_HOME under home, metrics and event flush off, and the +// migration freeze marker pointed at freezeMarker, a path that does not exist. +func discoveryPinEnv(home, freezeMarker string) []string { + var env []string + for _, kv := range testutil.WithoutGitRepoLocationEnv(os.Environ()) { + key, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(key, "BEADS_") || strings.HasPrefix(key, "BD_") || + key == "HOME" || key == "XDG_CONFIG_HOME" { + continue + } + env = append(env, kv) + } + return append(env, + "HOME="+home, + "XDG_CONFIG_HOME="+filepath.Join(home, ".config"), + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "BD_MIGRATION_FREEZE_FILE="+freezeMarker, + ) +} + +func (f *discoveryPinFixture) run(bd, cwd string, extraEnv []string, args ...string) (exit int, stdout, stderr string) { + f.t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = cwd + cmd.Env = append(discoveryPinEnv(f.path("home"), f.path("no-such-freeze-marker")), extraEnv...) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + if err := cmd.Run(); err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + f.t.Fatalf("run bd %s: %v", strings.Join(args, " "), err) + } + exit = exitErr.ExitCode() + } + return exit, f.normalize(out.String(), false), f.normalize(errOut.String(), true) +} + +var discoveryPinVersionLine = regexp.MustCompile(`(?m)^bd version .*$`) + +// discoveryPinDuplicateBinaries matches the warning bd version appends when +// the caller's PATH holds more than one distinct bd binary. +var discoveryPinDuplicateBinaries = regexp.MustCompile( + `\nWarning: multiple 'bd' binaries found in PATH:\n(?: [^\n]*\n)*` + + `The first one is being used\. Remove duplicates to avoid confusion\.\n`) + +// discoveryPinSandboxNotice matches the notice bd prints when it auto-detects +// a sandbox, which it does when it is not allowed to signal its own process. +var discoveryPinSandboxNotice = regexp.MustCompile(`(?m)^.*Sandbox detected, using direct mode\n`) + +// normalize rewrites the fixture root as $ROOT and the build's version line +// as "bd version $VERSION". For stderr it also drops the notices that depend +// on the caller's environment rather than on the fixture (sandbox +// auto-detection, and bd version's duplicate binaries in PATH) and collapses +// consecutive repeats of a "Warning: redirect" line: discovery re-resolves +// the same redirect several times per invocation and how many times it warns +// is not part of the pin. Any other repeated line, such as a doubled +// "Error:", is kept. +func (f *discoveryPinFixture) normalize(s string, stderr bool) string { + for _, form := range f.forms { + s = strings.ReplaceAll(s, form, "$ROOT") + } + s = discoveryPinVersionLine.ReplaceAllLiteralString(s, "bd version $VERSION") + if !stderr { + return s + } + s = discoveryPinDuplicateBinaries.ReplaceAllLiteralString(s, "") + s = discoveryPinSandboxNotice.ReplaceAllLiteralString(s, "") + var b strings.Builder + prev := "" + for _, line := range strings.SplitAfter(s, "\n") { + if line == prev && strings.HasPrefix(line, "Warning: redirect ") { + continue + } + b.WriteString(line) + prev = line + } + return b.String() +} + +// TestDiscoveryRefusalPinsMissingMetadata pins discovery when no workspace, +// a workspace without metadata.json and without a database, or a .beads that +// is a regular file is found. +func TestDiscoveryRefusalPinsMissingMetadata(t *testing.T) { + noBeads := func(f *discoveryPinFixture) (string, []string) { return f.project("p"), nil } + regularFileBeads := func(f *discoveryPinFixture) (string, []string) { + cwd := f.project("p") + f.write("p/.beads", "not a workspace\n") + return cwd, nil + } + runDiscoveryPinCases(t, []discoveryPinCase{ + { + name: "no-beads-dir/list", + setup: noBeads, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + // --json does not turn the refusal into a JSON document. + name: "no-beads-dir/list-json", + setup: noBeads, + args: []string{"list", "--json"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + name: "no-beads-dir/where", + setup: noBeads, + args: []string{"where"}, + exit: 1, + stderr: pinNoWorkspaceWhereRefusal, + }, + { + name: "no-beads-dir/where-json", + setup: noBeads, + args: []string{"where", "--json"}, + exit: 1, + stdout: pinNoWorkspaceWhereJSON, + }, + { + name: "empty-beads-dir/list", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.workspace("p", nil), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + name: "config-yaml-only/list", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.yamlWorkspace("p", nil), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + // The same directory is a workspace to bd where. + name: "config-yaml-only/where", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.yamlWorkspace("p", nil), nil + }, + args: []string{"where"}, + stdout: "$ROOT/p/.beads\n", + }, + { + // The working-directory walk selects a .beads that is a regular + // file, then fails to read its metadata.json (the BEADS_DIR form + // is in TestDiscoveryRefusalPinsExplicitBeadsDir). + name: "beads-is-regular-file/list", + setup: regularFileBeads, + args: []string{"list"}, + exit: 1, + stderr: "Error: loading config: reading config: open $ROOT/p/.beads/metadata.json: not a directory\n", + }, + { + name: "beads-is-regular-file/where", + setup: regularFileBeads, + args: []string{"where"}, + exit: 1, + stderr: "Error: loading config: reading config: open $ROOT/p/.beads/metadata.json: not a directory\n", + }, + }) +} + +// TestDiscoveryRefusalPinsCorruptMetadata pins the refusal for a discovered +// workspace whose metadata.json does not parse. +func TestDiscoveryRefusalPinsCorruptMetadata(t *testing.T) { + corrupt := func(f *discoveryPinFixture) (string, []string) { return f.corruptWorkspace("p", nil), nil } + runDiscoveryPinCases(t, []discoveryPinCase{ + { + name: "list", + setup: corrupt, + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + name: "list-json", + setup: corrupt, + args: []string{"list", "--json"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + name: "create", + setup: corrupt, + args: []string{"create", "must not be written", "--json"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + // bd where never opens a store, but it still refuses. + name: "where", + setup: corrupt, + args: []string{"where"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + name: "where-json", + setup: corrupt, + args: []string{"where", "--json"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + // bd version warns and succeeds. + name: "version", + setup: corrupt, + args: []string{"version"}, + stdout: "bd version $VERSION\n", + stderr: pinCorruptMetadataWarning("$ROOT/p/.beads"), + }, + }) +} + +// TestDiscoveryRefusalPinsExplicitBeadsDir pins an exported BEADS_DIR that +// names no workspace. Each case runs from a corrupt workspace, so falling +// back to discovery from the working directory would print that workspace's +// refusal instead. +func TestDiscoveryRefusalPinsExplicitBeadsDir(t *testing.T) { + from := func(beadsDir func(f *discoveryPinFixture) string) func(f *discoveryPinFixture) (string, []string) { + return func(f *discoveryPinFixture) (string, []string) { + cwd := f.corruptWorkspace("cwd", nil) + return cwd, []string{"BEADS_DIR=" + beadsDir(f)} + } + } + missing := from(func(f *discoveryPinFixture) string { return f.path("missing/.beads") }) + runDiscoveryPinCases(t, []discoveryPinCase{ + { + name: "missing-dir/list", + setup: missing, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + name: "missing-dir/where", + setup: missing, + args: []string{"where"}, + exit: 1, + stderr: pinNoWorkspaceWhereRefusal, + }, + { + name: "missing-dir/where-json", + setup: missing, + args: []string{"where", "--json"}, + exit: 1, + stdout: pinNoWorkspaceWhereJSON, + }, + { + name: "empty-dir/list", + setup: from(func(f *discoveryPinFixture) string { return f.mkdir("empty") }), + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + name: "regular-file/list", + setup: from(func(f *discoveryPinFixture) string { + f.write("file", "not a workspace\n") + return f.path("file") + }), + args: []string{"list"}, + exit: 1, + stderr: "Error: loading config: reading config: open $ROOT/file/metadata.json: not a directory\n", + }, + { + name: "corrupt-workspace/list", + setup: from(func(f *discoveryPinFixture) string { + f.corruptWorkspace("other", nil) + return f.path("other/.beads") + }), + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/other/.beads"), + }, + }) +} + +// TestDiscoveryRefusalPinsBeadsEnvFile pins settings supplied by the +// discovered workspace's .beads/.env. +func TestDiscoveryRefusalPinsBeadsEnvFile(t *testing.T) { + runDiscoveryPinCases(t, []discoveryPinCase{ + { + // A .env BEADS_DIR routes discovery to its target. + name: "beads-dir-routes-to-target", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + return f.yamlWorkspace("p", map[string]string{".env": "BEADS_DIR=" + f.path("target/.beads") + "\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + // The routing workspace's own corrupt metadata.json causes no refusal. + name: "beads-dir-route-skips-local-metadata", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", map[string]string{".env": "BEADS_DIR=" + f.path("missing/.beads") + "\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + // An exported BEADS_DIR wins; the .env is never read. + name: "exported-beads-dir-wins", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + cwd := f.yamlWorkspace("p", map[string]string{".env": "BEADS_DIR=" + f.path("target/.beads") + "\n"}) + return cwd, []string{"BEADS_DIR=" + f.path("missing/.beads")} + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + // A .env BEADS_DB does not route a data command away from the + // workspace that holds the .env (the shadowing + // TestDiscoveryRefusalPinsDBSelection records for an exported + // BEADS_DB) ... + name: "beads-db-list", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + return f.yamlWorkspace("p", map[string]string{".env": "BEADS_DB=" + f.path("target/.beads/dolt") + "\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + // ... but it does route bd where. + name: "beads-db-where", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + return f.yamlWorkspace("p", map[string]string{".env": "BEADS_DB=" + f.path("target/.beads/dolt") + "\n"}), nil + }, + args: []string{"where"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + // Settings in the selected workspace's .env apply (BD_JSON here). + name: "setting-applies-in-selected-workspace", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.yamlWorkspace("p", map[string]string{".env": "BD_JSON=true\n"}), nil + }, + args: []string{"where"}, + stdout: "{\n \"path\": \"$ROOT/p/.beads\",\n \"schema_version\": 1\n}\n", + }, + { + // A .env that routes elsewhere contributes only its selector: + // its other settings are not imported, so the refusal is text. + name: "setting-not-imported-from-routing-source", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.yamlWorkspace("p", map[string]string{".env": "BD_JSON=true\nBEADS_DIR=" + f.path("missing/.beads") + "\n"}), nil + }, + args: []string{"where"}, + exit: 1, + stderr: pinNoWorkspaceWhereRefusal, + }, + { + // checkBlockedEnvVars sees only the exported environment, and runs + // before any .env is loaded: a .env BD_BACKEND is not refused, here + // or anywhere before discovery's own no-database refusal. + name: "blocked-backend-not-checked", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.yamlWorkspace("p", map[string]string{".env": "BD_BACKEND=sqlite\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + }) +} + +// TestDiscoveryRefusalPinsRedirect pins .beads/redirect handling: the +// warnings for unusable targets and which directory is selected afterwards. +func TestDiscoveryRefusalPinsRedirect(t *testing.T) { + runDiscoveryPinCases(t, []discoveryPinCase{ + { + // A missing target is ignored and discovery keeps the source. + name: "missing-target", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", map[string]string{"redirect": f.path("missing/.beads") + "\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: "Warning: redirect target does not exist or is not a directory: $ROOT/missing/.beads\n" + + pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + // A target without project files is ignored the same way. + name: "target-without-workspace", + setup: func(f *discoveryPinFixture) (string, []string) { + f.mkdir("bare/.beads") + return f.corruptWorkspace("p", map[string]string{"redirect": "../bare/.beads\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: "Warning: ignoring redirect from $ROOT/p/.beads to $ROOT/bare/.beads because the target has no database or metadata.json; fix or delete the redirect file\n" + + pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + name: "corrupt-target", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + return f.workspace("p", map[string]string{"redirect": "../target/.beads\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + // Only one level is followed. + name: "chain", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("final", nil) + f.corruptWorkspace("middle", map[string]string{"redirect": "../final/.beads\n"}) + return f.workspace("p", map[string]string{"redirect": "../middle/.beads\n"}), nil + }, + args: []string{"list"}, + exit: 1, + stderr: "Warning: redirect chains not allowed, ignoring redirect in $ROOT/middle/.beads\n" + + pinCorruptMetadataRefusal("$ROOT/middle/.beads"), + }, + { + // An explicit BEADS_DIR is followed through its redirect. + name: "explicit-beads-dir", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + f.workspace("source", map[string]string{"redirect": "../target/.beads\n"}) + return f.project("p"), []string{"BEADS_DIR=" + f.path("source/.beads")} + }, + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + name: "where-reports-source", + setup: func(f *discoveryPinFixture) (string, []string) { + f.yamlWorkspace("target", nil) + return f.workspace("p", map[string]string{"redirect": "../target/.beads\n"}), nil + }, + args: []string{"where"}, + stdout: "$ROOT/target/.beads\n (via redirect from $ROOT/p/.beads)\n", + }, + }) +} + +// TestDiscoveryRefusalPinsDBSelection pins database selection by --db and by +// the BEADS_DB and BD_DB environment variables. +// +// Two of these record inconsistencies as they stand rather than endorse +// them: a data command ignores BD_DB, and lets a discoverable workspace in +// the working directory shadow BEADS_DB, while bd where honors both. +func TestDiscoveryRefusalPinsDBSelection(t *testing.T) { + target := func(f *discoveryPinFixture) string { + f.corruptWorkspace("target", nil) + return f.path("target/.beads/dolt") + } + runDiscoveryPinCases(t, []discoveryPinCase{ + { + // A --db path selects the workspace that owns it. + name: "db-path", + setup: func(f *discoveryPinFixture) (string, []string) { + target(f) + return f.project("p"), nil + }, + args: []string{"--db", "$ROOT/target/.beads/dolt", "list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + name: "db-path-beats-beads-dir", + setup: func(f *discoveryPinFixture) (string, []string) { + target(f) + f.corruptWorkspace("other", nil) + return f.project("p"), []string{"BEADS_DIR=" + f.path("other/.beads")} + }, + args: []string{"--db", "$ROOT/target/.beads/dolt", "list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + name: "db-path-beats-working-directory", + setup: func(f *discoveryPinFixture) (string, []string) { + target(f) + return f.corruptWorkspace("p", nil), nil + }, + args: []string{"--db", "$ROOT/target/.beads/dolt", "list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + // A --db value that is a database name rather than an existing + // path leaves discovery to the working directory. + name: "db-name-without-workspace", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), nil + }, + args: []string{"--db", "pindb", "list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + name: "db-name-in-workspace", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", nil), nil + }, + args: []string{"--db", "pindb", "list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + name: "beads-db-env", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BEADS_DB=" + target(f)} + }, + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + // Inconsistency: the working directory's workspace shadows BEADS_DB. + name: "beads-db-env-shadowed-by-working-directory", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", nil), []string{"BEADS_DB=" + target(f)} + }, + args: []string{"list"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/p/.beads"), + }, + { + name: "beads-db-env-where", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", nil), []string{"BEADS_DB=" + target(f)} + }, + args: []string{"where"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + { + // Inconsistency: a data command ignores BD_DB. + name: "bd-db-env", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_DB=" + target(f)} + }, + args: []string{"list"}, + exit: 1, + stderr: pinNoDatabaseRefusal, + }, + { + name: "bd-db-env-where", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_DB=" + target(f)} + }, + args: []string{"where"}, + exit: 1, + stderr: pinCorruptMetadataRefusal("$ROOT/target/.beads"), + }, + }) +} + +// TestDiscoveryRefusalPinsBlockedEnvVarsRunFirst pins that the blocked +// backend variables are refused before any discovery, for every command. +func TestDiscoveryRefusalPinsBlockedEnvVarsRunFirst(t *testing.T) { + runDiscoveryPinCases(t, []discoveryPinCase{ + { + name: "no-workspace", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_BACKEND=sqlite"} + }, + args: []string{"list"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + name: "corrupt-metadata", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", nil), []string{"BD_BACKEND=sqlite"} + }, + args: []string{"list"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + name: "explicit-beads-dir", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_BACKEND=sqlite", "BEADS_DIR=" + f.path("missing/.beads")} + }, + args: []string{"list"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + // A BEADS_DIR that cannot even be read as a directory is still + // reached only after the blocked variable is refused. + name: "unreadable-beads-dir", + setup: func(f *discoveryPinFixture) (string, []string) { + f.write("file", "not a workspace\n") + return f.project("p"), []string{"BD_BACKEND=sqlite", "BEADS_DIR=" + f.path("file")} + }, + args: []string{"list"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + name: "db-path", + setup: func(f *discoveryPinFixture) (string, []string) { + f.corruptWorkspace("target", nil) + return f.project("p"), []string{"BD_BACKEND=sqlite"} + }, + args: []string{"--db", "$ROOT/target/.beads/dolt", "list"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + // --json does not turn the refusal into a JSON document. + name: "database-backend-json", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.corruptWorkspace("p", nil), []string{"BD_DATABASE_BACKEND=dolt"} + }, + args: []string{"list", "--json"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_DATABASE_BACKEND"), + }, + { + name: "where-json", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_BACKEND=sqlite"} + }, + args: []string{"where", "--json"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + name: "version-command", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_BACKEND=sqlite"} + }, + args: []string{"version"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + }, + { + // The root command then prints cobra's error and usage; only the + // refusal itself is pinned. + name: "version-flag", + setup: func(f *discoveryPinFixture) (string, []string) { + return f.project("p"), []string{"BD_BACKEND=sqlite"} + }, + args: []string{"--version"}, + exit: 1, + stderr: pinBlockedEnvRefusal("BD_BACKEND"), + stderrPrefix: true, + }, + }) +} From eb5ab48950d9ab73aa0766e0105933ec2611586c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 05:19:28 +0000 Subject: [PATCH 23/54] test(setup): pin the Claude agent artifacts bd init and bd setup claude write (#7678) * test(setup): pin the Claude agent artifacts bd init and bd setup claude write Before-parity pin P11. Nothing pinned the agent artifacts byte for byte, so a change to them -- a new hook event in .claude/settings.json, new text in the minimal CLAUDE.md/AGENTS.md template and with it a new marker hash that marks every installed section stale -- could land without anyone seeing the user-visible diff. These goldens make any such change a reviewed diff. - internal/templates/agents: TestManagedSectionGolden pins each managed section (minimal and full profile, with and without a Dolt remote) markers included; TestManagedSectionHashes pins their marker hashes as hand-maintained literals; NoPush is pinned to render the no-remote bytes. - cmd/bd/setup: TestClaudeSetupArtifactsGolden records transcripts of bd setup claude, --check and --remove (project, --stealth, --global, a user's own settings and CLAUDE.md, a stale section, a legacy settings.local.json install, a CLAUDE.md that imports AGENTS.md): each step's result and output, and the settings and instruction files it creates, changes or deletes. TestClaudePluginManifestHookEvents pins the hook events and commands the beads Claude plugin declares. - cmd/bd: TestEmbeddedInitAgentArtifactsGolden runs a real bd init (default and --agents-profile full) and then bd setup claude --check, --remove and a reinstall through the binary, with exit codes. A managed section in a transcript is shown by its marker line and a pointer to the template golden it is checked to match byte for byte, so a template change shows up once in full and as a hash bump wherever it lands. Regenerate with BD_UPDATE_GOLDEN=1. CI companions: the new test files and testdata in BUILD.bazel (bd_test gains its internal/templates/agents dep), and the new TestEmbedded* in the 100-shard embedded cmd manifest (incremental --write). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(setup): drop BD_OTEL_* and split HOME in the agent artifact golden TestEmbeddedInitAgentArtifactsGolden pins bd's stderr byte for byte, but bdEnv passes the caller's BD_OTEL_* variables through, and bd prints a deprecation warning on stderr for a legacy one (BD_OTEL_METRICS_URL, BD_OTEL_LOGS_URL, BD_OTEL_STDOUT=true) before it checks whether telemetry is on. On a host that exports them (agent hosts do; CI does not), both scenarios failed at the first stderr block with no code drift. The test now runs bd init and every bd setup claude step with agentArtifactEnv: bdEnv(dir) without BD_OTEL_* and OTEL_*. It filters the env slice instead of calling clearTelemetryEnv, which uses t.Setenv and so cannot run in these parallel tests. The same helper gives bd a HOME of its own instead of bdEnv's HOME == project. With the two equal, a bd that wrote the hook and CLAUDE.md into the home directory instead of the project passed every pin. The transcript now also records $HOME/.claude/settings.json and $HOME/CLAUDE.md, and output paths under HOME read $HOME. bd init runs the steps bdInit does (git repo, seeded schema) with that env. The test doc now names what it does not pin: a proxied-server bd init (no Codex section, HasRemote from its own remote URL), a bd init with a remote, and the Cursor files. It also says a change to AGENTS.md's wrapper text or Codex section can fail only here. The goldens are unchanged. Validation (env -i, go1.26.7, GOFLAGS=-mod=readonly, GOPROXY=off, BEADS_TEST_EMBEDDED_DOLT=1, -tags gms_pure_go): - the golden passes in a clean env; with BD_OTEL_METRICS_URL=x BD_OTEL_LOGS_URL=y; with every legacy selector plus BD_OTEL_ENABLED and OTEL_EXPORTER_OTLP_ENDPOINT; and under -race; - the previous test with BD_OTEL_METRICS_URL=x BD_OTEL_LOGS_URL=y fails at golden line 132 on the deprecation warning; - swapping homeDir and projectDir in setup's defaultClaudeEnv passes the previous test and fails this one at golden line 5 (the transcript shows created $HOME/.claude/settings.json). Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- .github/scripts/embedded-cmd-test-shards.txt | 1 + cmd/bd/BUILD.bazel | 2 + cmd/bd/agent_artifacts_embedded_test.go | 306 +++++++++++ cmd/bd/setup/BUILD.bazel | 3 +- cmd/bd/setup/claude_artifacts_golden_test.go | 494 ++++++++++++++++++ .../claude-setup-agents-import-stub.golden | 84 +++ .../claude-setup-existing-user-files.golden | 174 ++++++ .../setup/testdata/claude-setup-global.golden | 122 +++++ .../claude-setup-legacy-local-settings.golden | 157 ++++++ .../claude-setup-project-stealth.golden | 122 +++++ .../testdata/claude-setup-project.golden | 142 +++++ .../claude-setup-stale-section.golden | 72 +++ .../init-agents-profile-full.golden | 133 +++++ .../agent_artifacts/init-default.golden | 274 ++++++++++ internal/templates/agents/BUILD.bazel | 2 + internal/templates/agents/golden_test.go | 133 +++++ .../managed-section-full-no-remote.md | 129 +++++ .../agents/testdata/managed-section-full.md | 131 +++++ .../managed-section-minimal-no-remote.md | 54 ++ .../testdata/managed-section-minimal.md | 55 ++ 20 files changed, 2589 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/agent_artifacts_embedded_test.go create mode 100644 cmd/bd/setup/claude_artifacts_golden_test.go create mode 100644 cmd/bd/setup/testdata/claude-setup-agents-import-stub.golden create mode 100644 cmd/bd/setup/testdata/claude-setup-existing-user-files.golden create mode 100644 cmd/bd/setup/testdata/claude-setup-global.golden create mode 100644 cmd/bd/setup/testdata/claude-setup-legacy-local-settings.golden create mode 100644 cmd/bd/setup/testdata/claude-setup-project-stealth.golden create mode 100644 cmd/bd/setup/testdata/claude-setup-project.golden create mode 100644 cmd/bd/setup/testdata/claude-setup-stale-section.golden create mode 100644 cmd/bd/testdata/agent_artifacts/init-agents-profile-full.golden create mode 100644 cmd/bd/testdata/agent_artifacts/init-default.golden create mode 100644 internal/templates/agents/golden_test.go create mode 100644 internal/templates/agents/testdata/managed-section-full-no-remote.md create mode 100644 internal/templates/agents/testdata/managed-section-full.md create mode 100644 internal/templates/agents/testdata/managed-section-minimal-no-remote.md create mode 100644 internal/templates/agents/testdata/managed-section-minimal.md diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 95d0ef0f0..7b45f7915 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -136,6 +136,7 @@ 100 54 TestEmbeddedDiscoveryPinsEnvFileReadonly 100 54 TestEmbeddedFormatJSONAliasMatchesJSON +100 54 TestEmbeddedInitAgentArtifactsGolden 100 54 TestEmbeddedSearchConcurrent 100 55 TestEmbeddedUndefer diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 3ac61dc7c..680301167 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -547,6 +547,7 @@ go_test( "actor_test.go", "ado_roundtrip_test.go", "ado_test.go", + "agent_artifacts_embedded_test.go", "agent_hook_test.go", "assign_fence_embedded_test.go", "assign_fence_proxied_test.go", @@ -1407,6 +1408,7 @@ go_test( "//internal/storage/uow", "//internal/storage/versioncontrolops", "//internal/telemetry", + "//internal/templates/agents", "//internal/testutil", "//internal/testutil/bazeltest", "//internal/testutil/credentialcmd", diff --git a/cmd/bd/agent_artifacts_embedded_test.go b/cmd/bd/agent_artifacts_embedded_test.go new file mode 100644 index 000000000..9636f9429 --- /dev/null +++ b/cmd/bd/agent_artifacts_embedded_test.go @@ -0,0 +1,306 @@ +//go:build cgo + +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/templates/agents" +) + +// TestEmbeddedInitAgentArtifactsGolden pins the Claude agent artifacts a real +// `bd init` writes — .claude/settings.json with its hook events and commands, +// CLAUDE.md and AGENTS.md with their managed sections — and what +// `bd setup claude --check`, `--remove` and a reinstall then report and leave +// behind, through the bd binary. Each scenario is a transcript golden under +// testdata/agent_artifacts/, so a change to any artifact or outcome (a new +// hook event, a changed command, new template text and its hash) shows up as +// a reviewed golden diff. +// +// A managed section is shown by its BEGIN marker line (profile and hash) and a +// pointer to the internal/templates/agents golden that pins its bytes +// (TestManagedSectionGolden), after checking the file holds exactly that +// section; one that matches no rendered variant is shown in full. The +// in-process `bd setup claude` scenarios are pinned by +// TestClaudeSetupArtifactsGolden in cmd/bd/setup. +// +// The scenarios run a direct (embedded) bd init with no Dolt remote. Not +// pinned here: a proxied-server bd init, which installs no Codex section and +// decides HasRemote from its own remote URL; a bd init with a remote, which +// renders the managed section's with-remote variant; and the Cursor files. +// AGENTS.md is shown in full apart from its managed section, so a change to +// its wrapper text or its Codex section can pass the cmd/bd/setup and +// internal/templates/agents tests and fail only here. +// +// bd runs with HOME outside the project, so an artifact written to the home +// directory instead of the project shows up under $HOME. +// +// Regenerate with +// +// BEADS_TEST_EMBEDDED_DOLT=1 BD_UPDATE_GOLDEN=1 go test -tags gms_pure_go ./cmd/bd/ -run TestEmbeddedInitAgentArtifactsGolden +func TestEmbeddedInitAgentArtifactsGolden(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + + scenarios := []struct { + name string + initArgs []string + steps [][]string + }{ + { + name: "init-default", + initArgs: []string{"--prefix", "aa"}, + steps: [][]string{ + {"setup", "claude", "--check"}, + {"setup", "claude"}, + {"setup", "claude", "--remove"}, + {"setup", "claude", "--check"}, + {"setup", "claude"}, + {"setup", "claude", "--check"}, + }, + }, + { + name: "init-agents-profile-full", + initArgs: []string{"--prefix", "ab", "--agents-profile", "full"}, + steps: [][]string{ + {"setup", "claude", "--check"}, + }, + }, + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + t.Parallel() + // bdInit, with HOME in a directory of its own. + dir, home := t.TempDir(), t.TempDir() + initGitRepoAt(t, dir) + database, seeded := seededInitDatabase(sc.initArgs) + if seeded { + seedEmbeddedSchema(t, dir, database) + } + initArgs := append([]string{"init", "--quiet"}, sc.initArgs...) + if stdout, stderr, exitCode := runAgentArtifactCommand(t, bd, dir, home, initArgs...); exitCode != 0 { + t.Fatalf("bd %s: exit %d\nstdout:\n%s\nstderr:\n%s", strings.Join(initArgs, " "), exitCode, stdout, stderr) + } + if seeded { + requireSeededEmbeddedInit(t, filepath.Join(dir, ".beads"), database) + } + + var transcript strings.Builder + fmt.Fprintf(&transcript, "# bd init agent artifact transcript: %s\n", sc.name) + fmt.Fprintf(&transcript, "# Regenerate with BEADS_TEST_EMBEDDED_DOLT=1 BD_UPDATE_GOLDEN=1 go test -tags gms_pure_go ./cmd/bd/ -run TestEmbeddedInitAgentArtifactsGolden\n") + fmt.Fprintf(&transcript, "\n=== bd %s\n", strings.Join(initArgs, " ")) + previous := map[string]string{} + writeAgentArtifactChanges(t, &transcript, dir, home, previous) + + for _, args := range sc.steps { + stdout, stderr, exitCode := runAgentArtifactCommand(t, bd, dir, home, args...) + fmt.Fprintf(&transcript, "\n=== bd %s\n", strings.Join(args, " ")) + fmt.Fprintf(&transcript, "exit: %d\n", exitCode) + writeAgentArtifactBlock(&transcript, "stdout", normalizeAgentArtifactOutput(stdout, dir, home)) + writeAgentArtifactBlock(&transcript, "stderr", normalizeAgentArtifactOutput(stderr, dir, home)) + writeAgentArtifactChanges(t, &transcript, dir, home, previous) + } + + assertAgentArtifactGolden(t, filepath.Join("testdata", "agent_artifacts", sc.name+".golden"), transcript.String()) + }) + } +} + +// agentArtifactPaths are the Claude agent artifacts bd init and bd setup +// claude write in a project ($PROJECT), and the two a global install writes +// in the home directory ($HOME) instead. +var agentArtifactPaths = []string{ + "$PROJECT/.claude/settings.json", + "$PROJECT/.claude/settings.local.json", + "$PROJECT/AGENTS.md", + "$PROJECT/CLAUDE.md", + "$HOME/.claude/settings.json", + "$HOME/CLAUDE.md", +} + +// agentArtifactManagedSections maps each managed section bd renders to the +// internal/templates/agents golden file that pins its bytes. +var agentArtifactManagedSections = []struct { + golden string + profile agents.Profile + opts agents.RenderOpts +}{ + {golden: "managed-section-minimal.md", profile: agents.ProfileMinimal, opts: agents.RenderOpts{HasRemote: true}}, + {golden: "managed-section-minimal-no-remote.md", profile: agents.ProfileMinimal, opts: agents.RenderOpts{}}, + {golden: "managed-section-full.md", profile: agents.ProfileFull, opts: agents.RenderOpts{HasRemote: true}}, + {golden: "managed-section-full-no-remote.md", profile: agents.ProfileFull, opts: agents.RenderOpts{}}, +} + +// elideAgentArtifactManagedSection replaces the body of a managed section +// that is byte-identical to a rendered variant with a pointer to that +// variant's golden, keeping the BEGIN marker line (profile and hash) and the +// END marker. +func elideAgentArtifactManagedSection(content string) string { + const endMarker = "\n" + begin := strings.Index(content, "\n" + begin := strings.Index(content, "\nOlder beads guidance.\n\n\n## Notes\n", + }, + steps: []claudeArtifactStep{ + claudeCheckStep(), + claudeInstallStep(false, false), + claudeCheckStep(), + }, + }, + { + // A legacy install in settings.local.json: install migrates + // its hook to settings.json. + name: "legacy-local-settings", + seed: map[string]string{ + "project/.claude/settings.local.json": legacyLocalSettings, + }, + steps: []claudeArtifactStep{ + claudeCheckStep(), + claudeInstallStep(false, false), + claudeCheckStep(), + claudeRemoveStep(false), + }, + }, + { + // CLAUDE.md imports AGENTS.md: the managed section goes to + // AGENTS.md, and --remove leaves that shared section in place. + name: "agents-import-stub", + seed: map[string]string{ + "project/AGENTS.md": "# Agent Instructions\n\nShared by every agent.\n", + "project/CLAUDE.md": "@AGENTS.md\n", + }, + steps: []claudeArtifactStep{ + claudeInstallStep(false, false), + claudeCheckStep(), + claudeRemoveStep(false), + claudeCheckStep(), + }, + }, + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + root := filepath.Dir(env.projectDir) + for rel, content := range sc.seed { + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir for seed %s: %v", rel, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("seed %s: %v", rel, err) + } + } + + var transcript strings.Builder + fmt.Fprintf(&transcript, "# bd setup claude artifact transcript: %s\n", sc.name) + fmt.Fprintf(&transcript, "# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden\n") + previous := map[string]string{} + writeClaudeArtifactChanges(t, &transcript, "seed", root, previous) + for _, step := range sc.steps { + out, errOut, err := runClaudeArtifactStep(t, env, step.run) + fmt.Fprintf(&transcript, "\n=== %s\n", step.title) + fmt.Fprintf(&transcript, "result: %s\n", claudeStepResult(err)) + writeTranscriptBlock(&transcript, "stdout", normalizeArtifactOutput(out, root)) + writeTranscriptBlock(&transcript, "stderr", normalizeArtifactOutput(errOut, root)) + writeClaudeArtifactChanges(t, &transcript, "files", root, previous) + } + + assertArtifactGolden(t, filepath.Join("testdata", "claude-setup-"+sc.name+".golden"), transcript.String()) + }) + } +} + +// TestClaudePluginManifestHookEvents pins the hook events and commands the +// beads Claude Code plugin declares. With the plugin enabled bd setup claude +// writes no hooks of its own (GH#3192), so these are the hooks such a user +// runs. +func TestClaudePluginManifestHookEvents(t *testing.T) { + var manifest struct { + Hooks map[string][]struct { + Matcher string `json:"matcher"` + Hooks []struct { + Type string `json:"type"` + Command string `json:"command"` + } `json:"hooks"` + } `json:"hooks"` + } + readJSONFile(t, filepath.Join("..", "..", "..", "plugins", "beads", ".claude-plugin", "plugin.json"), &manifest) + + var got []string + for event, entries := range manifest.Hooks { + for _, entry := range entries { + for _, hook := range entry.Hooks { + got = append(got, fmt.Sprintf("%s matcher=%q %s: %s", event, entry.Matcher, hook.Type, hook.Command)) + } + } + } + sort.Strings(got) + want := []string{ + `PreCompact matcher="" command: bd prime`, + `SessionStart matcher="" command: bd prime`, + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("Claude plugin hooks changed:\ngot:\n %s\nwant:\n %s", strings.Join(got, "\n "), strings.Join(want, "\n ")) + } +} + +// runClaudeArtifactStep runs one step with env.stdout and the process stdout +// sent to the same file, so the transcript keeps the order a user sees: the +// hook helpers (addHookCommand, removeHookCommand) print with fmt.Printf +// rather than to env.stdout. Swapping os.Stdout is why these tests must not +// run in parallel. +func runClaudeArtifactStep(t *testing.T, env claudeEnv, run func(claudeEnv) error) (stdout, stderr string, err error) { + t.Helper() + capture, createErr := os.CreateTemp(t.TempDir(), "stdout-*") + if createErr != nil { + t.Fatalf("create stdout capture: %v", createErr) + } + errBuf := &bytes.Buffer{} + env.stdout, env.stderr = capture, errBuf + + origStdout := os.Stdout + os.Stdout = capture + func() { + defer func() { os.Stdout = origStdout }() + err = run(env) + }() + + if closeErr := capture.Close(); closeErr != nil { + t.Fatalf("close stdout capture: %v", closeErr) + } + data, readErr := os.ReadFile(capture.Name()) + if readErr != nil { + t.Fatalf("read stdout capture: %v", readErr) + } + return string(data), errBuf.String(), err +} + +func claudeStepResult(err error) string { + if err == nil { + return "ok" + } + return "error: " + err.Error() +} + +// writeClaudeArtifactChanges appends every artifact file created, modified or +// deleted since previous (which it then updates), with its full contents. +func writeClaudeArtifactChanges(t *testing.T, b *strings.Builder, label, root string, previous map[string]string) { + t.Helper() + changed := false + for _, rel := range claudeArtifactPaths { + data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + before, existed := previous[rel] + switch { + case os.IsNotExist(err): + if existed { + fmt.Fprintf(b, "--- %s: deleted $ROOT/%s\n", label, rel) + delete(previous, rel) + changed = true + } + case err != nil: + t.Fatalf("read %s: %v", rel, err) + case !existed || before != string(data): + verb := "created" + if existed { + verb = "modified" + } + shown := string(data) + if strings.HasSuffix(rel, ".md") { + shown = elideManagedSection(shown) + } + writeTranscriptBlock(b, fmt.Sprintf("%s: %s $ROOT/%s", label, verb, rel), shown) + previous[rel] = string(data) + changed = true + } + } + switch { + case changed: + case label == "seed": + fmt.Fprintf(b, "--- %s: none\n", label) + default: + fmt.Fprintf(b, "--- %s: unchanged\n", label) + } +} + +// writeTranscriptBlock appends a titled block, marking empty content and a +// missing trailing newline so neither is lost in the golden. +func writeTranscriptBlock(b *strings.Builder, title, content string) { + fmt.Fprintf(b, "--- %s\n", title) + switch { + case content == "": + b.WriteString("(empty)\n") + case !strings.HasSuffix(content, "\n"): + b.WriteString(content) + b.WriteString("\n(no newline at end of file)\n") + default: + b.WriteString(content) + } +} + +// normalizeArtifactOutput replaces the test's temporary root with $ROOT and +// uses forward slashes, so the transcript is the same on every host. +func normalizeArtifactOutput(s, root string) string { + s = strings.ReplaceAll(s, root, "$ROOT") + if filepath.Separator != '/' { + s = strings.ReplaceAll(s, string(filepath.Separator), "/") + } + return s +} + +// assertArtifactGolden compares got with the golden file at path, or rewrites +// the file when BD_UPDATE_GOLDEN=1. +func assertArtifactGolden(t *testing.T, path, got string) { + t.Helper() + if os.Getenv("BD_UPDATE_GOLDEN") == "1" { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(got), 0o644); err != nil { + t.Fatalf("write golden %s: %v", path, err) + } + t.Logf("updated golden %s", path) + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s (regenerate with BD_UPDATE_GOLDEN=1): %v", path, err) + } + if got == string(want) { + return + } + wantLines, gotLines := strings.Split(string(want), "\n"), strings.Split(got, "\n") + line := func(lines []string, i int) string { + if i >= len(lines) { + return "(end of output)" + } + return fmt.Sprintf("%q", lines[i]) + } + i := 0 + for i < len(wantLines) && i < len(gotLines) && wantLines[i] == gotLines[i] { + i++ + } + t.Errorf("Claude agent artifacts drifted from %s (regenerate with BD_UPDATE_GOLDEN=1 if intended)\nfirst difference at line %d:\n want: %s\n got: %s\n--- got ---\n%s", path, i+1, line(wantLines, i), line(gotLines, i), got) +} diff --git a/cmd/bd/setup/testdata/claude-setup-agents-import-stub.golden b/cmd/bd/setup/testdata/claude-setup-agents-import-stub.golden new file mode 100644 index 000000000..9b4337a80 --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-agents-import-stub.golden @@ -0,0 +1,84 @@ +# bd setup claude artifact transcript: agents-import-stub +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: created $ROOT/project/AGENTS.md +# Agent Instructions + +Shared by every agent. +--- seed: created $ROOT/project/CLAUDE.md +@AGENTS.md + +=== bd setup claude +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Registered SessionStart hook +Installing Claude Code integration... +✓ Added beads section to existing AGENTS.md + +✓ Claude Code integration installed + File: $ROOT/project/AGENTS.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: created $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: modified $ROOT/project/AGENTS.md +# Agent Instructions + +Shared by every agent. + + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + +=== bd setup claude --check +result: ok +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/AGENTS.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove +result: ok +--- stdout +Removing Claude hooks from project... +✓ Removed SessionStart hook + Leaving shared beads section in AGENTS.md untouched (project-authoritative, not Claude-specific) +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $ROOT/project/.claude/settings.json +{ + "hooks": {} +} + +=== bd setup claude --check +result: error: claude hooks not installed +--- stdout +✗ No hooks installed + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/setup/testdata/claude-setup-existing-user-files.golden b/cmd/bd/setup/testdata/claude-setup-existing-user-files.golden new file mode 100644 index 000000000..d81091fb8 --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-existing-user-files.golden @@ -0,0 +1,174 @@ +# bd setup claude artifact transcript: existing-user-files +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: created $ROOT/project/.claude/settings.json +{ + "permissions": { + "allow": [ + "Bash(go test:*)" + ] + }, + "hooks": { + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "echo session started" + } + ] + } + ], + "Stop": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "notify-send done" + } + ] + } + ] + } +} +--- seed: created $ROOT/project/CLAUDE.md +# My Project + +House rules live here. + +=== bd setup claude +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Registered SessionStart hook +Installing Claude Code integration... +✓ Added beads section to existing CLAUDE.md + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: modified $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "echo session started", + "type": "command" + } + ], + "matcher": "" + }, + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ], + "Stop": [ + { + "hooks": [ + { + "command": "notify-send done", + "type": "command" + } + ], + "matcher": "" + } + ] + }, + "permissions": { + "allow": [ + "Bash(go test:*)" + ] + } +} +--- files: modified $ROOT/project/CLAUDE.md +# My Project + +House rules live here. + + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + +=== bd setup claude --check +result: ok +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove +result: ok +--- stdout +Removing Claude hooks from project... +✓ Removed SessionStart hook +Removing Claude Code integration... +✓ Removed beads section from CLAUDE.md +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "echo session started", + "type": "command" + } + ], + "matcher": "" + } + ], + "Stop": [ + { + "hooks": [ + { + "command": "notify-send done", + "type": "command" + } + ], + "matcher": "" + } + ] + }, + "permissions": { + "allow": [ + "Bash(go test:*)" + ] + } +} +--- files: modified $ROOT/project/CLAUDE.md +# My Project + +House rules live here. + + + +=== bd setup claude --check +result: error: claude hooks not installed +--- stdout +✗ No hooks installed + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/setup/testdata/claude-setup-global.golden b/cmd/bd/setup/testdata/claude-setup-global.golden new file mode 100644 index 000000000..da63d077c --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-global.golden @@ -0,0 +1,122 @@ +# bd setup claude artifact transcript: global +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: none + +=== bd setup claude --global +result: ok +--- stdout +Installing Claude hooks globally... +✓ Registered SessionStart hook +Installing Claude Code integration... +✓ Created new CLAUDE.md with beads integration + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/home/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: created $ROOT/home/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: created $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: ok +--- stdout +✓ Global hooks installed: $ROOT/home/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove --global +result: ok +--- stdout +Removing Claude hooks globally... +✓ Removed SessionStart hook +Removing Claude Code integration... +✓ Removed beads section from CLAUDE.md +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $ROOT/home/.claude/settings.json +{ + "hooks": {} +} +--- files: modified $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: error: claude hooks not installed +--- stdout +✗ No hooks installed + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/setup/testdata/claude-setup-legacy-local-settings.golden b/cmd/bd/setup/testdata/claude-setup-legacy-local-settings.golden new file mode 100644 index 000000000..1e17ea55a --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-legacy-local-settings.golden @@ -0,0 +1,157 @@ +# bd setup claude artifact transcript: legacy-local-settings +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: created $ROOT/project/.claude/settings.local.json +{ + "hooks": { + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "bd prime" + } + ] + } + ], + "PreCompact": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "bd prime" + } + ] + } + ] + } +} + +=== bd setup claude --check +result: error: agents file not found +--- stdout +✓ Project hooks installed (legacy): $ROOT/project/.claude/settings.local.json + Consider running 'bd setup claude' to migrate to .claude/settings.json +✗ CLAUDE.md not found + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Registered SessionStart hook +✓ Removed SessionStart hook +✓ Removed PreCompact hook +✓ Migrated hooks from $ROOT/project/.claude/settings.local.json +Installing Claude Code integration... +✓ Created new CLAUDE.md with beads integration + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: created $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: modified $ROOT/project/.claude/settings.local.json +{ + "hooks": {} +} +--- files: created $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: ok +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove +result: ok +--- stdout +Removing Claude hooks from project... +✓ Removed SessionStart hook +Removing Claude Code integration... +✓ Removed beads section from CLAUDE.md +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $ROOT/project/.claude/settings.json +{ + "hooks": {} +} +--- files: modified $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ diff --git a/cmd/bd/setup/testdata/claude-setup-project-stealth.golden b/cmd/bd/setup/testdata/claude-setup-project-stealth.golden new file mode 100644 index 000000000..71d62d8e6 --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-project-stealth.golden @@ -0,0 +1,122 @@ +# bd setup claude artifact transcript: project-stealth +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: none + +=== bd setup claude --stealth +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Registered SessionStart hook +Installing Claude Code integration... +✓ Created new CLAUDE.md with beads integration + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: created $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --stealth --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: created $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: ok +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove +result: ok +--- stdout +Removing Claude hooks from project... +✓ Removed SessionStart hook +Removing Claude Code integration... +✓ Removed beads section from CLAUDE.md +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $ROOT/project/.claude/settings.json +{ + "hooks": {} +} +--- files: modified $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: error: claude hooks not installed +--- stdout +✗ No hooks installed + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/setup/testdata/claude-setup-project.golden b/cmd/bd/setup/testdata/claude-setup-project.golden new file mode 100644 index 000000000..aa307d08f --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-project.golden @@ -0,0 +1,142 @@ +# bd setup claude artifact transcript: project +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: none + +=== bd setup claude +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Registered SessionStart hook +Installing Claude Code integration... +✓ Created new CLAUDE.md with beads integration + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: created $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: created $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: ok +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Hook already registered: SessionStart +Installing Claude Code integration... +✓ Updated existing beads section in CLAUDE.md + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove +result: ok +--- stdout +Removing Claude hooks from project... +✓ Removed SessionStart hook +Removing Claude Code integration... +✓ Removed beads section from CLAUDE.md +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $ROOT/project/.claude/settings.json +{ + "hooks": {} +} +--- files: modified $ROOT/project/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +result: error: claude hooks not installed +--- stdout +✗ No hooks installed + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/setup/testdata/claude-setup-stale-section.golden b/cmd/bd/setup/testdata/claude-setup-stale-section.golden new file mode 100644 index 000000000..c6be8e135 --- /dev/null +++ b/cmd/bd/setup/testdata/claude-setup-stale-section.golden @@ -0,0 +1,72 @@ +# bd setup claude artifact transcript: stale-section +# Regenerate with BD_UPDATE_GOLDEN=1 go test ./cmd/bd/setup/ -run TestClaudeSetupArtifactsGolden +--- seed: created $ROOT/project/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- seed: created $ROOT/project/CLAUDE.md +# My Project + + +Older beads guidance. + + +## Notes + +=== bd setup claude --check +result: error: beads section is stale +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +⚠ Claude Code integration installed but stale: $ROOT/project/CLAUDE.md + Run: bd setup claude (to update) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude +result: ok +--- stdout +Installing Claude hooks for this project... +✓ Hook already registered: SessionStart +Installing Claude Code integration... +✓ Updated existing beads section in CLAUDE.md + +✓ Claude Code integration installed + File: $ROOT/project/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $ROOT/project/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: modified $ROOT/project/CLAUDE.md +# My Project + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal.md] + + +## Notes + +=== bd setup claude --check +result: ok +--- stdout +✓ Project hooks installed: $ROOT/project/.claude/settings.json +✓ Claude Code integration installed: $ROOT/project/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/testdata/agent_artifacts/init-agents-profile-full.golden b/cmd/bd/testdata/agent_artifacts/init-agents-profile-full.golden new file mode 100644 index 000000000..e7abe0511 --- /dev/null +++ b/cmd/bd/testdata/agent_artifacts/init-agents-profile-full.golden @@ -0,0 +1,133 @@ +# bd init agent artifact transcript: init-agents-profile-full +# Regenerate with BEADS_TEST_EMBEDDED_DOLT=1 BD_UPDATE_GOLDEN=1 go test -tags gms_pure_go ./cmd/bd/ -run TestEmbeddedInitAgentArtifactsGolden + +=== bd init --quiet --prefix ab --agents-profile full +--- files: created $PROJECT/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: created $PROJECT/AGENTS.md +# Agent Instructions + +This project uses **bd** (beads) for issue tracking. Run `bd prime` for full workflow context. + +> **Architecture in one line:** Issues live in a local Dolt database +> (`.beads/dolt/`); cross-machine sync uses `bd dolt push/pull` (a +> git-compatible protocol), stored under `refs/dolt/data` on your git +> remote — separate from `refs/heads/*` where your code lives. +> `.beads/issues.jsonl` is a passive export, not the wire protocol. +> +> See [sync-concepts](https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md) +> for the one-screen overview and anti-patterns (don't treat JSONL as the +> source of truth; don't `bd import` during normal operation; don't +> reach for third-party Dolt hosting before trying the default). + +## Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work atomically +bd close # Complete work +bd dolt push # Push beads data to remote +``` + +## Non-Interactive Shell Commands + +**ALWAYS use non-interactive flags** with file operations to avoid hanging on confirmation prompts. + +Shell commands like `cp`, `mv`, and `rm` may be aliased to include `-i` (interactive) mode on some systems, causing the agent to hang indefinitely waiting for y/n input. + +**Use these forms instead:** +```bash +# Force overwrite without prompting +cp -f source dest # NOT: cp source dest +mv -f source dest # NOT: mv source dest +rm -f file # NOT: rm file + +# For recursive operations +rm -rf directory # NOT: rm -r directory +cp -rf source dest # NOT: cp -r source dest +``` + +**Other commands that may prompt:** +- `scp` - use `-o BatchMode=yes` for non-interactive +- `ssh` - use `-o BatchMode=yes` to fail instead of prompting +- `apt-get` - use `-y` flag +- `brew` - use `HOMEBREW_NO_AUTO_UPDATE=1` env var + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-full-no-remote.md] + + + +## Beads Issue Tracker + +Use Beads (`bd`) for durable task tracking in repositories that include it. Use the `beads` skill at `.agents/skills/beads/SKILL.md` (project install) or `~/.agents/skills/beads/SKILL.md` (global install) for Beads workflow guidance, then use the `bd` CLI for issue operations. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +bd prime # Refresh Beads context +``` + +### Rules + +- Use `bd` for all task tracking; do not create markdown TODO lists. +- Run `bd prime` when Beads context is missing or stale. Codex 0.129.0+ can load Beads context automatically through native hooks; use `/hooks` to inspect or toggle them. +- Keep durable project facts in Beads via `bd remember` rather than in per-tool memory files, which fragment across accounts; per-operator preferences belong in your harness's own memory. + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. + +--- files: created $PROJECT/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal-no-remote.md] + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +exit: 0 +--- stdout +✓ Project hooks installed: $PROJECT/.claude/settings.json +✓ Claude Code integration installed: $PROJECT/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged diff --git a/cmd/bd/testdata/agent_artifacts/init-default.golden b/cmd/bd/testdata/agent_artifacts/init-default.golden new file mode 100644 index 000000000..6d3929882 --- /dev/null +++ b/cmd/bd/testdata/agent_artifacts/init-default.golden @@ -0,0 +1,274 @@ +# bd init agent artifact transcript: init-default +# Regenerate with BEADS_TEST_EMBEDDED_DOLT=1 BD_UPDATE_GOLDEN=1 go test -tags gms_pure_go ./cmd/bd/ -run TestEmbeddedInitAgentArtifactsGolden + +=== bd init --quiet --prefix aa +--- files: created $PROJECT/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: created $PROJECT/AGENTS.md +# Agent Instructions + +This project uses **bd** (beads) for issue tracking. Run `bd prime` for full workflow context. + +> **Architecture in one line:** Issues live in a local Dolt database +> (`.beads/dolt/`); cross-machine sync uses `bd dolt push/pull` (a +> git-compatible protocol), stored under `refs/dolt/data` on your git +> remote — separate from `refs/heads/*` where your code lives. +> `.beads/issues.jsonl` is a passive export, not the wire protocol. +> +> See [sync-concepts](https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md) +> for the one-screen overview and anti-patterns (don't treat JSONL as the +> source of truth; don't `bd import` during normal operation; don't +> reach for third-party Dolt hosting before trying the default). + +## Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work atomically +bd close # Complete work +bd dolt push # Push beads data to remote +``` + +## Non-Interactive Shell Commands + +**ALWAYS use non-interactive flags** with file operations to avoid hanging on confirmation prompts. + +Shell commands like `cp`, `mv`, and `rm` may be aliased to include `-i` (interactive) mode on some systems, causing the agent to hang indefinitely waiting for y/n input. + +**Use these forms instead:** +```bash +# Force overwrite without prompting +cp -f source dest # NOT: cp source dest +mv -f source dest # NOT: mv source dest +rm -f file # NOT: rm file + +# For recursive operations +rm -rf directory # NOT: rm -r directory +cp -rf source dest # NOT: cp -r source dest +``` + +**Other commands that may prompt:** +- `scp` - use `-o BatchMode=yes` for non-interactive +- `ssh` - use `-o BatchMode=yes` to fail instead of prompting +- `apt-get` - use `-y` flag +- `brew` - use `HOMEBREW_NO_AUTO_UPDATE=1` env var + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal-no-remote.md] + + + +## Beads Issue Tracker + +Use Beads (`bd`) for durable task tracking in repositories that include it. Use the `beads` skill at `.agents/skills/beads/SKILL.md` (project install) or `~/.agents/skills/beads/SKILL.md` (global install) for Beads workflow guidance, then use the `bd` CLI for issue operations. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +bd prime # Refresh Beads context +``` + +### Rules + +- Use `bd` for all task tracking; do not create markdown TODO lists. +- Run `bd prime` when Beads context is missing or stale. Codex 0.129.0+ can load Beads context automatically through native hooks; use `/hooks` to inspect or toggle them. +- Keep durable project facts in Beads via `bd remember` rather than in per-tool memory files, which fragment across accounts; per-operator preferences belong in your harness's own memory. + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. + +--- files: created $PROJECT/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal-no-remote.md] + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +exit: 0 +--- stdout +✓ Project hooks installed: $PROJECT/.claude/settings.json +✓ Claude Code integration installed: $PROJECT/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude +exit: 0 +--- stdout +Installing Claude hooks for this project... +✓ Hook already registered: SessionStart +Installing Claude Code integration... +✓ Updated existing beads section in CLAUDE.md + +✓ Claude Code integration installed + File: $PROJECT/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $PROJECT/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude --remove +exit: 0 +--- stdout +Removing Claude hooks from project... +✓ Removed SessionStart hook +Removing Claude Code integration... +✓ Removed beads section from CLAUDE.md +✓ Claude hooks removed +--- stderr +(empty) +--- files: modified $PROJECT/.claude/settings.json +{ + "hooks": {} +} +--- files: modified $PROJECT/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + +=== bd setup claude --check +exit: 1 +--- stdout +✗ No hooks installed + Run: bd setup claude +--- stderr +(empty) +--- files: unchanged + +=== bd setup claude +exit: 0 +--- stdout +Installing Claude hooks for this project... +✓ Registered SessionStart hook +Installing Claude Code integration... +✓ Added beads section to existing CLAUDE.md + +✓ Claude Code integration installed + File: $PROJECT/CLAUDE.md +No additional configuration needed! + +✓ Claude Code integration installed + Settings: $PROJECT/.claude/settings.json + +Restart Claude Code for changes to take effect. +--- stderr +(empty) +--- files: modified $PROJECT/.claude/settings.json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime --hook-json", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} +--- files: modified $PROJECT/CLAUDE.md +# Project Instructions for AI Agents + +This file provides instructions and context for AI coding agents working on this project. + + + +## Build & Test + +_Add your build and test commands here_ + +```bash +# Example: +# npm install +# npm test +``` + +## Architecture Overview + +_Add a brief overview of your project architecture_ + +## Conventions & Patterns + +_Add your project-specific conventions here_ + + + +[managed section body: byte-identical to internal/templates/agents/testdata/managed-section-minimal-no-remote.md] + + +=== bd setup claude --check +exit: 0 +--- stdout +✓ Project hooks installed: $PROJECT/.claude/settings.json +✓ Claude Code integration installed: $PROJECT/CLAUDE.md (current) +--- stderr +(empty) +--- files: unchanged diff --git a/internal/templates/agents/BUILD.bazel b/internal/templates/agents/BUILD.bazel index c51ae9c90..8ad3255df 100644 --- a/internal/templates/agents/BUILD.bazel +++ b/internal/templates/agents/BUILD.bazel @@ -20,8 +20,10 @@ go_test( name = "agents_test", srcs = [ "agents_test.go", + "golden_test.go", "render_test.go", ], + data = glob(["testdata/**"]), embed = [":agents"], ) diff --git a/internal/templates/agents/golden_test.go b/internal/templates/agents/golden_test.go new file mode 100644 index 000000000..e49b0acf8 --- /dev/null +++ b/internal/templates/agents/golden_test.go @@ -0,0 +1,133 @@ +package agents + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// The tests in this file pin the exact managed sections `bd init` and +// `bd setup ` write between the BEGIN/END BEADS INTEGRATION markers of +// CLAUDE.md, AGENTS.md and the other instruction files. A change to a template +// (new guidance, a reworded rule, an added section) changes the bytes every +// user's instruction file receives and, through the marker hash, marks every +// section already installed as stale: `bd setup claude --check` then fails and +// the next `bd init` or `bd setup` rewrites the user's file. Such a change must +// therefore arrive as a reviewed golden diff, never as a side effect. +// +// Regenerate the golden files with +// +// BD_UPDATE_GOLDEN=1 go test ./internal/templates/agents/ -run TestManagedSectionGolden +// +// and update the literal hashes in TestManagedSectionHashes by hand. + +// managedSectionVariant is one managed section bd can render. +type managedSectionVariant struct { + name string + profile Profile + opts RenderOpts +} + +// managedSectionVariants lists every managed section bd renders: both +// profiles, each with and without the `bd dolt push` lines that depend on a +// configured Dolt remote. NoPush renders the no-remote bytes +// (TestManagedSectionNoPushMatchesNoRemote), so it needs no variant of its own. +var managedSectionVariants = []managedSectionVariant{ + {name: "minimal", profile: ProfileMinimal, opts: RenderOpts{HasRemote: true}}, + {name: "minimal-no-remote", profile: ProfileMinimal, opts: RenderOpts{}}, + {name: "full", profile: ProfileFull, opts: RenderOpts{HasRemote: true}}, + {name: "full-no-remote", profile: ProfileFull, opts: RenderOpts{}}, +} + +// TestManagedSectionGolden pins each managed section byte for byte, markers +// included. +func TestManagedSectionGolden(t *testing.T) { + for _, v := range managedSectionVariants { + t.Run(v.name, func(t *testing.T) { + assertGoldenFile(t, filepath.Join("testdata", "managed-section-"+v.name+".md"), RenderSectionWithOpts(v.profile, v.opts)) + }) + } +} + +// TestManagedSectionHashes pins the marker hash of each managed section. The +// hash is what `bd setup --check` compares against an installed +// section, so changing it marks every existing install stale. It is a literal +// that BD_UPDATE_GOLDEN does not rewrite: a template change has to bump it by +// hand. +func TestManagedSectionHashes(t *testing.T) { + want := map[string]string{ + "minimal": "6ffdf896", + "minimal-no-remote": "cdcea925", + "full": "bacef91e", + "full-no-remote": "9c890b20", + } + for _, v := range managedSectionVariants { + got := CurrentHashWithOpts(v.profile, v.opts) + if got != want[v.name] { + t.Errorf("%s managed section hash = %q, want %q: the template changed, so every installed %s section is now stale", v.name, got, want[v.name], v.profile) + } + marker := "\n" + if section := RenderSectionWithOpts(v.profile, v.opts); !strings.HasPrefix(section, marker) { + first, _, _ := strings.Cut(section, "\n") + t.Errorf("%s managed section marker = %q, want %q", v.name, first, strings.TrimSuffix(marker, "\n")) + } + } +} + +// TestManagedSectionNoPushMatchesNoRemote pins that a local-only workspace +// (no-push: true) gets the same managed section as one with no Dolt remote, +// whatever the remote state. +func TestManagedSectionNoPushMatchesNoRemote(t *testing.T) { + for _, profile := range []Profile{ProfileMinimal, ProfileFull} { + want := RenderSectionWithOpts(profile, RenderOpts{}) + for _, hasRemote := range []bool{false, true} { + if got := RenderSectionWithOpts(profile, RenderOpts{HasRemote: hasRemote, NoPush: true}); got != want { + t.Errorf("%s profile with NoPush (HasRemote=%v) differs from the no-remote section", profile, hasRemote) + } + } + } +} + +// assertGoldenFile compares got with the golden file at path, or rewrites the +// file when BD_UPDATE_GOLDEN=1. +func assertGoldenFile(t *testing.T, path, got string) { + t.Helper() + if os.Getenv("BD_UPDATE_GOLDEN") == "1" { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(got), 0o644); err != nil { + t.Fatalf("write golden %s: %v", path, err) + } + t.Logf("updated golden %s", path) + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s (regenerate with BD_UPDATE_GOLDEN=1): %v", path, err) + } + if got == string(want) { + return + } + t.Errorf("rendered output drifted from %s (regenerate with BD_UPDATE_GOLDEN=1 if intended)\n%s", path, firstGoldenDifference(string(want), got)) +} + +// firstGoldenDifference describes the first line on which want and got differ. +func firstGoldenDifference(want, got string) string { + wantLines := strings.Split(want, "\n") + gotLines := strings.Split(got, "\n") + line := func(lines []string, i int) string { + if i >= len(lines) { + return "(end of output)" + } + return fmt.Sprintf("%q", lines[i]) + } + for i := 0; i < len(wantLines) || i < len(gotLines); i++ { + if i >= len(wantLines) || i >= len(gotLines) || wantLines[i] != gotLines[i] { + return fmt.Sprintf("first difference at line %d:\n want: %s\n got: %s", i+1, line(wantLines, i), line(gotLines, i)) + } + } + return "outputs differ" +} diff --git a/internal/templates/agents/testdata/managed-section-full-no-remote.md b/internal/templates/agents/testdata/managed-section-full-no-remote.md new file mode 100644 index 000000000..526550030 --- /dev/null +++ b/internal/templates/agents/testdata/managed-section-full-no-remote.md @@ -0,0 +1,129 @@ + +## Issue Tracking with bd (beads) + +**IMPORTANT**: This project uses **bd (beads)** for ALL issue tracking. Do NOT use markdown TODOs, task lists, or other tracking methods. + +### Why bd? + +- Dependency-aware: Track blockers and relationships between issues +- Git-friendly: Dolt-powered version control with native sync +- Agent-optimized: JSON output, ready work detection, discovered-from links +- Prevents duplicate tracking systems and confusion + +### Quick Start + +**Check for ready work:** + +```bash +bd ready --json +``` + +**Create new issues:** + +```bash +bd create "Issue title" --description="Detailed context" -t bug|feature|task -p 0-4 --json +bd create "Issue title" --description="What this issue is about" -p 1 --deps discovered-from:bd-123 --json +``` + +**Claim and update:** + +```bash +bd update --claim --json +bd update bd-42 --priority 1 --json +``` + +**Complete work:** + +```bash +bd close bd-42 --reason "Completed" --json +``` + +### Issue Types + +- `bug` - Something broken +- `feature` - New functionality +- `task` - Work item (tests, docs, refactoring) +- `epic` - Large feature with subtasks +- `chore` - Maintenance (dependencies, tooling) + +### Priorities + +- `0` - Critical (security, data loss, broken builds) +- `1` - High (major features, important bugs) +- `2` - Medium (default, nice-to-have) +- `3` - Low (polish, optimization) +- `4` - Backlog (future ideas) + +### Workflow for AI Agents + +1. **Check ready work**: `bd ready` shows unblocked issues +2. **Claim your task atomically**: `bd update --claim` +3. **Work on it**: Implement, test, document +4. **Discover new work?** Create linked issue: + - `bd create "Found bug" --description="Details about what was found" -p 1 --deps discovered-from:` +5. **Complete**: `bd close --reason "Done"` + +### Quality +- Use `--acceptance` and `--design` fields when creating issues +- Use `--validate` to check description completeness + +### Lifecycle +- `bd defer ` / `bd supersede ` for issue management +- `bd stale` / `bd orphans` / `bd lint` for hygiene +- `bd human ` to flag for human decisions +- `bd formula list` / `bd mol pour ` for structured workflows + +### Sync + +bd stores issue history in Dolt: + +- Each write auto-commits to Dolt history +- Do not treat `.beads/issues.jsonl` as the sync protocol + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. + +### Important Rules + +- ✅ Use bd for ALL task tracking +- ✅ Always use `--json` flag for programmatic use +- ✅ Link discovered work with `discovered-from` dependencies +- ✅ Check `bd ready` before asking "what should I work on?" +- ❌ Do NOT create markdown TODO lists +- ❌ Do NOT use external issue trackers +- ❌ Do NOT duplicate tracking systems + +For more details, see README.md and https://github.com/gastownhall/beads/blob/main/docs/getting-started/quickstart.md. + +## Agent Context Profiles + +The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. + +- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. +- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. +- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. + +## Session Completion + +This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. + +1. **File issues for remaining work** - Create beads for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **Handle git/sync by active profile**: + ```bash + # Conservative/minimal/default: report status and proposed commands; wait for approval. + git status + + # Team-maintainer opt-in only, unless current instructions forbid it: + git pull --rebase + git push + git status + ``` +5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step + +**Critical rules:** +- Explicit user or orchestrator instructions override this Beads block. +- Do not commit or push without clear authority from the active profile or the current user request. +- If a required sync or push is blocked, stop and report the exact command and error. + + diff --git a/internal/templates/agents/testdata/managed-section-full.md b/internal/templates/agents/testdata/managed-section-full.md new file mode 100644 index 000000000..831c2b736 --- /dev/null +++ b/internal/templates/agents/testdata/managed-section-full.md @@ -0,0 +1,131 @@ + +## Issue Tracking with bd (beads) + +**IMPORTANT**: This project uses **bd (beads)** for ALL issue tracking. Do NOT use markdown TODOs, task lists, or other tracking methods. + +### Why bd? + +- Dependency-aware: Track blockers and relationships between issues +- Git-friendly: Dolt-powered version control with native sync +- Agent-optimized: JSON output, ready work detection, discovered-from links +- Prevents duplicate tracking systems and confusion + +### Quick Start + +**Check for ready work:** + +```bash +bd ready --json +``` + +**Create new issues:** + +```bash +bd create "Issue title" --description="Detailed context" -t bug|feature|task -p 0-4 --json +bd create "Issue title" --description="What this issue is about" -p 1 --deps discovered-from:bd-123 --json +``` + +**Claim and update:** + +```bash +bd update --claim --json +bd update bd-42 --priority 1 --json +``` + +**Complete work:** + +```bash +bd close bd-42 --reason "Completed" --json +``` + +### Issue Types + +- `bug` - Something broken +- `feature` - New functionality +- `task` - Work item (tests, docs, refactoring) +- `epic` - Large feature with subtasks +- `chore` - Maintenance (dependencies, tooling) + +### Priorities + +- `0` - Critical (security, data loss, broken builds) +- `1` - High (major features, important bugs) +- `2` - Medium (default, nice-to-have) +- `3` - Low (polish, optimization) +- `4` - Backlog (future ideas) + +### Workflow for AI Agents + +1. **Check ready work**: `bd ready` shows unblocked issues +2. **Claim your task atomically**: `bd update --claim` +3. **Work on it**: Implement, test, document +4. **Discover new work?** Create linked issue: + - `bd create "Found bug" --description="Details about what was found" -p 1 --deps discovered-from:` +5. **Complete**: `bd close --reason "Done"` + +### Quality +- Use `--acceptance` and `--design` fields when creating issues +- Use `--validate` to check description completeness + +### Lifecycle +- `bd defer ` / `bd supersede ` for issue management +- `bd stale` / `bd orphans` / `bd lint` for hygiene +- `bd human ` to flag for human decisions +- `bd formula list` / `bd mol pour ` for structured workflows + +### Sync + +bd stores issue history in Dolt: + +- Each write auto-commits to Dolt history +- Use `bd dolt push`/`bd dolt pull` for remote sync +- Do not treat `.beads/issues.jsonl` as the sync protocol + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. + +### Important Rules + +- ✅ Use bd for ALL task tracking +- ✅ Always use `--json` flag for programmatic use +- ✅ Link discovered work with `discovered-from` dependencies +- ✅ Check `bd ready` before asking "what should I work on?" +- ❌ Do NOT create markdown TODO lists +- ❌ Do NOT use external issue trackers +- ❌ Do NOT duplicate tracking systems + +For more details, see README.md and https://github.com/gastownhall/beads/blob/main/docs/getting-started/quickstart.md. + +## Agent Context Profiles + +The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. + +- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. +- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. +- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. + +## Session Completion + +This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. + +1. **File issues for remaining work** - Create beads for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **Handle git/sync by active profile**: + ```bash + # Conservative/minimal/default: report status and proposed commands; wait for approval. + git status + + # Team-maintainer opt-in only, unless current instructions forbid it: + git pull --rebase + bd dolt push + git push + git status + ``` +5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step + +**Critical rules:** +- Explicit user or orchestrator instructions override this Beads block. +- Do not commit or push without clear authority from the active profile or the current user request. +- If a required sync or push is blocked, stop and report the exact command and error. + + diff --git a/internal/templates/agents/testdata/managed-section-minimal-no-remote.md b/internal/templates/agents/testdata/managed-section-minimal-no-remote.md new file mode 100644 index 000000000..1ab497bff --- /dev/null +++ b/internal/templates/agents/testdata/managed-section-minimal-no-remote.md @@ -0,0 +1,54 @@ + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for durable project facts — not per-tool memory files, which fragment across accounts; keep per-operator preferences in your harness's own memory + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. + +## Agent Context Profiles + +The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. + +- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. +- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. +- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. + +## Session Completion + +This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. + +1. **File issues for remaining work** - Create beads for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **Handle git/sync by active profile**: + ```bash + # Conservative/minimal/default: report status and proposed commands; wait for approval. + git status + + # Team-maintainer opt-in only, unless current instructions forbid it: + git pull --rebase + git push + git status + ``` +5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step + +**Critical rules:** +- Explicit user or orchestrator instructions override this Beads block. +- Do not commit or push without clear authority from the active profile or the current user request. +- If a required sync or push is blocked, stop and report the exact command and error. + diff --git a/internal/templates/agents/testdata/managed-section-minimal.md b/internal/templates/agents/testdata/managed-section-minimal.md new file mode 100644 index 000000000..22c48412a --- /dev/null +++ b/internal/templates/agents/testdata/managed-section-minimal.md @@ -0,0 +1,55 @@ + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for durable project facts — not per-tool memory files, which fragment across accounts; keep per-operator preferences in your harness's own memory + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. + +## Agent Context Profiles + +The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. + +- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. +- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. +- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. + +## Session Completion + +This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. + +1. **File issues for remaining work** - Create beads for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **Handle git/sync by active profile**: + ```bash + # Conservative/minimal/default: report status and proposed commands; wait for approval. + git status + + # Team-maintainer opt-in only, unless current instructions forbid it: + git pull --rebase + bd dolt push + git push + git status + ``` +5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step + +**Critical rules:** +- Explicit user or orchestrator instructions override this Beads block. +- Do not commit or push without clear authority from the active profile or the current user request. +- If a required sync or push is blocked, stop and report the exact command and error. + From 36f81bab1cd7ae915debb0add694f291203506ca Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 05:56:15 +0000 Subject: [PATCH 24/54] test(cmd/bd): pin the command tree and representative --help output (#7675) * test(cmd/bd): pin the command tree and representative --help output Add two in-process goldens for the bd command surface, so any added, removed, hidden or reworded command or flag fails until the goldens are regenerated and shows up as a reviewed testdata diff: - TestCommandTreeGolden walks the whole cobra tree (hidden commands included) and records, per command, Use, Aliases, Short, Long, Example, group, hidden/deprecated state, runnability and annotations, plus every local and persistent flag it defines (name, shorthand, type, default, usage, hidden, deprecation, annotations). Output is sorted and rendered twice to prove it is deterministic. - TestCommandHelpGolden runs `bd --help` through rootCmd for root, create, show, list, update, close, version, versions, config and serve. --help returns before the root pre-run, so no store is opened; color escapes are stripped. The snapshots are byte-identical to a built bd binary's --help output. Both run from a fresh ordinary (dependency-mode) workspace: a .beads/metadata.json with no graph marker and no database behind it. The tree is primed the way the binary is (main()'s help command and its --all/--doc/--list/--docs-root flags, ExecuteC's completion command), and cobra's history-dependent pieces (the lazily added per-command --help flag, the __complete request commands) are excluded, so the result does not depend on which in-process tests ran first. The help runs reset every flag afterwards so a later in-process command does not inherit --help. docs/cli-docs.pin holds check-doc-flags at the v1.2.2 release, so drift on main is not caught there; these goldens are the live guard. They pin the cgo build (the !cgo build swaps in a federation stub). Regenerate with BD_UPDATE_GOLDEN=1 go test -tags gms_pure_go ./cmd/bd -run '^TestCommand(Tree|Help)Golden$'. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * test(cmd/bd): keep the command goldens from merging flags into the shared tree TestCommandTreeGolden read each command's flags through LocalNonPersistentFlags, and TestCommandHelpGolden dispatches `bd serve --help`. Both run cobra's mergePersistentFlags, which copies every inherited persistent flag (--json, --db, ...) into the command's own flag set for the rest of the process. TestServeFlags pins serveCmd's exact flag set, so it failed whenever a golden test ran before it in the same test binary: tree then serve flags, help then serve flags, and -test.shuffle seeds 2 and 3, on the PR head and on its merge with main. CI passes only because bd_test's 16 Bazel shards happen to run them in different processes. Of 37 candidate tests, TestServeFlags is the only one affected. - goldenLocalFlags reads the set LocalNonPersistentFlags reports without calling it: c.Flags() minus c's own persistent flags and minus inherited copies. Like cobra, it treats a flag as inherited only if it is the nearest ancestor's own *pflag.Flag, so copies left by an earlier in-process execution are skipped and a flag that shadows an inherited one still renders. - TestCommandTreeGolden fails if rendering changes the number of flags in any command's flag set. - TestCommandHelpGolden calls restoreServeGlobals, which re-registers serveCmd's flags after the `bd serve --help` dispatch. Review minors: - The regeneration command (doc comment, failure-message constant, tree.golden header) sets CGO_ENABLED=1. Without cgo the file does not build, so the command matches no test and exits 0. - enterOrdinaryWorkspaceForGolden calls isolateBeadsDirForTest. - renderCommandHelpGolden's comment says when the ANSI strip matters. - cmd/bd/AGENTS.md lists the goldens under "Adding a command" and "Guard tests". Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Eddie the Engineer <338826946+gascity-eddie[bot]@users.noreply.github.com> --- cmd/bd/AGENTS.md | 7 + cmd/bd/BUILD.bazel | 1 + cmd/bd/command_tree_golden_test.go | 444 + cmd/bd/testdata/command_tree/help/bd.golden | 163 + .../command_tree/help/bd_close.golden | 57 + .../command_tree/help/bd_config.golden | 122 + .../command_tree/help/bd_create.golden | 72 + .../testdata/command_tree/help/bd_list.golden | 89 + .../command_tree/help/bd_serve.golden | 119 + .../testdata/command_tree/help/bd_show.golden | 41 + .../command_tree/help/bd_update.golden | 78 + .../command_tree/help/bd_version.golden | 24 + .../command_tree/help/bd_versions.golden | 60 + cmd/bd/testdata/command_tree/tree.golden | 7461 +++++++++++++++++ 14 files changed, 8738 insertions(+) create mode 100644 cmd/bd/command_tree_golden_test.go create mode 100644 cmd/bd/testdata/command_tree/help/bd.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_close.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_config.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_create.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_list.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_serve.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_show.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_update.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_version.golden create mode 100644 cmd/bd/testdata/command_tree/help/bd_versions.golden create mode 100644 cmd/bd/testdata/command_tree/tree.golden diff --git a/cmd/bd/AGENTS.md b/cmd/bd/AGENTS.md index 5ddbc95f5..fcaf8bef8 100644 --- a/cmd/bd/AGENTS.md +++ b/cmd/bd/AGENTS.md @@ -38,6 +38,10 @@ Which tier a test belongs in, and which command runs it, is decided by 6. Regenerate the CLI reference with `scripts/generate-cli-docs.sh`; `scripts/check-cli-docs-drift.sh` fails a PR whose command-surface change leaves the generated docs stale. +7. Regenerate the command-surface goldens and review the testdata diff: + `BD_UPDATE_GOLDEN=1 CGO_ENABLED=1 go test -tags gms_pure_go ./cmd/bd -run '^TestCommand(Tree|Help)Golden$'`. + `TestCommandTreeGolden` fails on any added, removed or reworded command or + flag until you do. Prefer a flag on an existing command over a new command, and route recovery/repair operations to `bd doctor --fix`; see the CLI design principles @@ -85,6 +89,9 @@ no linter enforces it). or serialize the call under `stdioMutex`. - `TestCorpusGolden` (`protocol/`) — no unreviewed change to the `--json` wire shapes. +- `TestCommandTreeGolden` and `TestCommandHelpGolden` — no unreviewed change + to the command tree (every command and flag, hidden ones included) or to + the pinned `--help` output in `testdata/command_tree/`. Fresh-workspace command fixtures call `isolateBeadsDirForTest(t)` before setup or dispatch and must not use `t.Parallel()`. diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 680301167..cfff7fd67 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -605,6 +605,7 @@ go_test( "cobra_flag_reset_test.go", "codex_hook_test.go", "command_telemetry_test.go", + "command_tree_golden_test.go", "comment_promote_embedded_test.go", "comment_proxied_integration_test.go", "comment_reserved_word_test.go", diff --git a/cmd/bd/command_tree_golden_test.go b/cmd/bd/command_tree_golden_test.go new file mode 100644 index 000000000..b22600883 --- /dev/null +++ b/cmd/bd/command_tree_golden_test.go @@ -0,0 +1,444 @@ +//go:build cgo + +// cgo only: the goldens pin the cgo build, which every lane that runs this +// package's full suite compiles. The !cgo build replaces the federation +// subtree with a one-command stub (federation_nocgo.go), so its tree differs +// by design. + +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/charmbracelet/x/ansi" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// The committed command-surface goldens. Regenerate after an intentional CLI +// change, then review the diff like code: +// +// BD_UPDATE_GOLDEN=1 CGO_ENABLED=1 go test -tags gms_pure_go ./cmd/bd -run '^TestCommand(Tree|Help)Golden$' +// +// CGO_ENABLED=1 because this file builds only with cgo: with cgo off the +// command matches no test, exits 0 and leaves the goldens as they were. +const ( + commandTreeGoldenPath = "testdata/command_tree/tree.golden" + commandHelpGoldenDir = "testdata/command_tree/help" + commandGoldenRegen = "BD_UPDATE_GOLDEN=1 CGO_ENABLED=1 go test -tags gms_pure_go ./cmd/bd -run '^TestCommand(Tree|Help)Golden$'" +) + +// commandHelpGoldenArgs is the representative set of commands whose exact +// --help output is pinned, each as the argv after "bd". The tree golden +// covers every command's metadata; these cover the rendering (usage line, +// flag table layout, inherited global flags, root command groups). +var commandHelpGoldenArgs = [][]string{ + {}, + {"create"}, + {"show"}, + {"list"}, + {"update"}, + {"close"}, + {"version"}, + {"versions"}, + {"config"}, + {"serve"}, +} + +// TestCommandTreeGolden pins the whole bd command tree: for every command and +// subcommand, hidden ones included, its Use, Aliases, Short, Long, Example, +// group, hidden/deprecated state, runnability and annotations, and every +// local and persistent flag it defines (name, shorthand, type, default, +// usage, hidden, deprecation, annotations). Any added, removed or reworded +// command or flag fails this test until the golden is regenerated, so the +// change shows up as a reviewed testdata diff. It reads the in-memory tree +// only: no store or subprocess. +// +// docs/cli-docs.pin pins the generated CLI reference (and check-doc-flags) +// to a release, so drift on main is not caught there; this golden is the +// live guard for the command surface. +func TestCommandTreeGolden(t *testing.T) { + pkgDir := enterOrdinaryWorkspaceForGolden(t) + primeCommandTreeForGolden() + flagCounts := commandFlagCounts(rootCmd) + got := renderCommandTreeGolden(rootCmd) + // A renderer that iterated a map would pass on the run that regenerated + // the golden and flake afterwards. + if again := renderCommandTreeGolden(rootCmd); again != got { + t.Fatalf("command tree rendering is not deterministic:\n%s", goldenLineDiff(got, again)) + } + // Nor may rendering change the tree it reads: tests sharing this binary + // pin a command's exact flag set (TestServeFlags). See goldenLocalFlags. + if after := commandFlagCounts(rootCmd); after != flagCounts { + t.Fatalf("rendering the command tree added flags to a command's own flag set:\n%s", goldenLineDiff(flagCounts, after)) + } + checkCommandGolden(t, pkgDir, commandTreeGoldenPath, got) +} + +// TestCommandHelpGolden pins the exact `bd --help` output of a +// representative set of commands, executed in-process through the real root +// command. --help returns before the root pre-run, so no store is opened. +// Color escapes are stripped so the snapshot is independent of terminal +// detection; otherwise the snapshots are byte-identical to what a built bd +// prints. +func TestCommandHelpGolden(t *testing.T) { + pkgDir := enterOrdinaryWorkspaceForGolden(t) + // Parsing `bd serve --help` merges the root's persistent flags into + // serveCmd's own flag set, which TestServeFlags pins; restoreServeGlobals + // re-registers that set afterwards. + restoreServeGlobals(t) + + want := map[string]bool{} + for _, args := range commandHelpGoldenArgs { + name := strings.Join(append([]string{"bd"}, args...), "_") + file := name + ".golden" + want[file] = true + t.Run(name, func(t *testing.T) { + checkCommandGolden(t, pkgDir, filepath.Join(commandHelpGoldenDir, file), renderCommandHelpGolden(t, args)) + }) + } + + // The directory holds exactly the listed snapshots: a command dropped + // from the list must not leave a stale file that still looks pinned. + dir := filepath.Join(pkgDir, commandHelpGoldenDir) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", commandHelpGoldenDir, err) + } + for _, e := range entries { + if want[e.Name()] { + continue + } + if os.Getenv("BD_UPDATE_GOLDEN") == "1" { + if err := os.Remove(filepath.Join(dir, e.Name())); err != nil { + t.Fatalf("remove stale help golden %s: %v", e.Name(), err) + } + t.Logf("removed stale help golden %s", e.Name()) + continue + } + t.Errorf("%s/%s is not in commandHelpGoldenArgs; delete it or regenerate with %s", commandHelpGoldenDir, e.Name(), commandGoldenRegen) + } +} + +// enterOrdinaryWorkspaceForGolden moves the test into a fresh ordinary +// (dependency-mode) workspace: a .beads/metadata.json with no graph marker +// and no database behind it, so nothing can be read from or written to a +// store. It returns the package directory, where the goldens live. +func enterOrdinaryWorkspaceForGolden(t *testing.T) string { + t.Helper() + isolateBeadsDirForTest(t) + pkgDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + ws := t.TempDir() + beadsDir := filepath.Join(ws, ".beads") + if err := os.Mkdir(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Chdir(ws) + return pkgDir +} + +// primeCommandTreeForGolden completes the tree the way the bd binary does +// before it dispatches: main() adds cobra's help command and its +// --all/--doc/--list/--docs-root flags, and ExecuteC adds the completion +// command. All three are idempotent, so the tree is the same whether or not +// an earlier in-process test already executed rootCmd. +func primeCommandTreeForGolden() { + rootCmd.InitDefaultHelpCmd() + registerHelpAllFlag() + rootCmd.InitDefaultCompletionCmd() +} + +// renderCommandTreeGolden walks cmd's tree depth first, children sorted by +// name, and renders one block per command. +func renderCommandTreeGolden(root *cobra.Command) string { + var b strings.Builder + b.WriteString("# bd command tree (cgo build), pinned by TestCommandTreeGolden.\n") + b.WriteString("# Regenerate after an intentional CLI change:\n") + b.WriteString("# " + commandGoldenRegen + "\n") + + var walk func(*cobra.Command) + walk = func(c *cobra.Command) { + writeCommandGoldenBlock(&b, c) + children := append([]*cobra.Command(nil), c.Commands()...) + sort.Slice(children, func(i, j int) bool { return children[i].Name() < children[j].Name() }) + for _, sub := range children { + // cobra adds the hidden shell-completion request commands only + // for an execution that invokes them, and then never removes + // them: whether they are present depends on test history, not + // on the bd surface. + if sub.Name() == cobra.ShellCompRequestCmd || sub.Name() == cobra.ShellCompNoDescRequestCmd { + continue + } + walk(sub) + } + } + walk(root) + return b.String() +} + +func writeCommandGoldenBlock(b *strings.Builder, c *cobra.Command) { + fmt.Fprintf(b, "\n%s\n", c.CommandPath()) + fmt.Fprintf(b, " use: %q\n", c.Use) + if len(c.Aliases) > 0 { + fmt.Fprintf(b, " aliases: %q\n", c.Aliases) + } + fmt.Fprintf(b, " short: %q\n", c.Short) + if c.GroupID != "" { + fmt.Fprintf(b, " group: %q\n", c.GroupID) + } + fmt.Fprintf(b, " hidden: %t\n", c.Hidden) + if c.Deprecated != "" { + fmt.Fprintf(b, " deprecated: %q\n", c.Deprecated) + } + fmt.Fprintf(b, " runnable: %t\n", c.Runnable()) + if c.DisableFlagParsing { + b.WriteString(" disable-flag-parsing: true\n") + } + if len(c.ValidArgs) > 0 { + fmt.Fprintf(b, " valid-args: %q\n", c.ValidArgs) + } + if len(c.Annotations) > 0 { + fmt.Fprintf(b, " annotations: %s\n", formatGoldenAnnotations(c.Annotations)) + } + writeGoldenTextBlock(b, "long", c.Long) + writeGoldenTextBlock(b, "example", c.Example) + for _, line := range commandGoldenFlagLines(c.PersistentFlags()) { + fmt.Fprintf(b, " persistent-flag %s\n", line) + } + for _, line := range commandGoldenFlagLines(goldenLocalFlags(c)) { + fmt.Fprintf(b, " flag %s\n", line) + } +} + +// goldenLocalFlags returns the non-persistent flags c registers itself, +// including any that shadow an inherited flag: the set +// c.LocalNonPersistentFlags() reports, read without calling it. That +// accessor, like LocalFlags and InheritedFlags, runs cobra's +// mergePersistentFlags, which copies every inherited persistent flag into +// c.Flags() for the rest of the process. Over the whole tree that would leave +// `bd serve` carrying the root's --json, --db and the rest, and break +// TestServeFlags (see TestNoSubcommandShadowsRootJSONFlag). +// +// c.Flags() may already hold such copies from an earlier in-process +// execution. They are recognized by identity, the way cobra itself tells +// them apart: an inherited flag is the nearest ancestor's own *pflag.Flag of +// that name, and a shadowing flag is a different one. +func goldenLocalFlags(c *cobra.Command) *pflag.FlagSet { + local := pflag.NewFlagSet(c.Name(), pflag.ContinueOnError) + c.Flags().VisitAll(func(f *pflag.Flag) { + if c.PersistentFlags().Lookup(f.Name) != nil { + return // rendered as a persistent-flag line + } + for p := c.Parent(); p != nil; p = p.Parent() { + if inherited := p.PersistentFlags().Lookup(f.Name); inherited != nil { + if inherited == f { + return + } + break + } + } + local.AddFlag(f) + }) + return local +} + +// commandFlagCounts renders how many flags each command's own flag set holds, +// one line per command, read without merging anything into it. +func commandFlagCounts(root *cobra.Command) string { + var b strings.Builder + var walk func(*cobra.Command) + walk = func(c *cobra.Command) { + n := 0 + c.Flags().VisitAll(func(*pflag.Flag) { n++ }) + fmt.Fprintf(&b, "%s %d\n", c.CommandPath(), n) + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(root) + return b.String() +} + +// writeGoldenTextBlock writes a multi-line field one source line per golden +// line, behind a "|" gutter so leading whitespace and blank lines survive. +func writeGoldenTextBlock(b *strings.Builder, name, text string) { + if text == "" { + return + } + fmt.Fprintf(b, " %s:\n", name) + for _, line := range strings.Split(text, "\n") { + if line == "" { + b.WriteString(" |\n") + continue + } + fmt.Fprintf(b, " | %s\n", line) + } +} + +// commandGoldenFlagLines renders each flag of fs on one line, sorted by name. +// cobra's own lazily added --help flag (FlagSetByCobraAnnotation) is skipped: +// it exists only on commands an earlier in-process execution reached. +func commandGoldenFlagLines(fs *pflag.FlagSet) []string { + var flags []*pflag.Flag + fs.VisitAll(func(f *pflag.Flag) { + if _, byCobra := f.Annotations[cobra.FlagSetByCobraAnnotation]; byCobra { + return + } + flags = append(flags, f) + }) + sort.Slice(flags, func(i, j int) bool { return flags[i].Name < flags[j].Name }) + + lines := make([]string, 0, len(flags)) + for _, f := range flags { + var b strings.Builder + fmt.Fprintf(&b, "--%s", f.Name) + if f.Shorthand != "" { + fmt.Fprintf(&b, " -%s", f.Shorthand) + } + fmt.Fprintf(&b, " type=%s default=%q", f.Value.Type(), f.DefValue) + if f.NoOptDefVal != "" && !implicitNoOptDefault(f) { + fmt.Fprintf(&b, " no-opt-default=%q", f.NoOptDefVal) + } + if f.Hidden { + b.WriteString(" hidden") + } + if f.Deprecated != "" { + fmt.Fprintf(&b, " deprecated=%q", f.Deprecated) + } + if f.ShorthandDeprecated != "" { + fmt.Fprintf(&b, " shorthand-deprecated=%q", f.ShorthandDeprecated) + } + if len(f.Annotations) > 0 { + fmt.Fprintf(&b, " annotations=%s", formatGoldenAnnotations(f.Annotations)) + } + fmt.Fprintf(&b, " usage=%q", f.Usage) + lines = append(lines, b.String()) + } + return lines +} + +// implicitNoOptDefault reports whether f's NoOptDefVal is the one pflag gives +// every flag of its type (a bare --bool means true, a bare --count adds one), +// which would only repeat the type on each line. +func implicitNoOptDefault(f *pflag.Flag) bool { + switch f.Value.Type() { + case "bool": + return f.NoOptDefVal == "true" + case "count": + return f.NoOptDefVal == "+1" + } + return false +} + +// formatGoldenAnnotations renders a command's or a flag's annotations sorted +// by key. +func formatGoldenAnnotations[V string | []string](m map[string]V) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%q", k, m[k])) + } + return "{" + strings.Join(parts, " ") + "}" +} + +// renderCommandHelpGolden runs `bd --help` through rootCmd in-process +// and returns its stdout with color escapes stripped. cobra keeps the parsed +// --help value on the shared tree, so every flag is put back afterwards: a +// later in-process `bd create ...` must not print help instead. +// +// internal/ui decides color once, at init, from CLICOLOR_FORCE and whether the +// test binary's stdout is a terminal. CI never gives it one, so there the +// strip removes nothing; it is what keeps a run from a terminal matching the +// goldens. +func renderCommandHelpGolden(t *testing.T, args []string) string { + t.Helper() + argv := append(append([]string(nil), args...), "--help") + primeCommandTreeForGolden() + var execErr error + out := captureStdout(t, func() error { + rootCmd.SetArgs(argv) + defer rootCmd.SetArgs(nil) + defer resetCommandFlags(rootCmd) + _, execErr = rootCmd.ExecuteC() + return nil + }) + if execErr != nil { + t.Fatalf("bd %s: %v", strings.Join(argv, " "), execErr) + } + return ansi.Strip(out) +} + +// checkCommandGolden compares got with the golden at rel under pkgDir, or +// rewrites the golden when BD_UPDATE_GOLDEN=1 (the package's shared golden +// switch). +func checkCommandGolden(t *testing.T, pkgDir, rel, got string) { + t.Helper() + path := filepath.Join(pkgDir, rel) + if os.Getenv("BD_UPDATE_GOLDEN") == "1" { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(rel), err) + } + if err := os.WriteFile(path, []byte(got), 0o644); err != nil { + t.Fatalf("write golden %s: %v", rel, err) + } + t.Logf("updated golden %s", rel) + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s (regenerate with %s): %v", rel, commandGoldenRegen, err) + } + if bytes.Equal([]byte(got), want) { + return + } + t.Errorf("bd command surface drifted from %s.\n%s\nIf the change is intended, regenerate with\n %s\nand review the testdata diff.", + rel, goldenLineDiff(string(want), got), commandGoldenRegen) +} + +// goldenLineDiff reports the region where want and got differ: the common +// leading and trailing lines are trimmed and the rest is shown as -/+ lines, +// capped so a large drift stays readable (the reviewed artifact is the +// regenerated testdata diff, not this message). +func goldenLineDiff(want, got string) string { + const maxLines = 40 + w := strings.Split(want, "\n") + g := strings.Split(got, "\n") + pre := 0 + for pre < len(w) && pre < len(g) && w[pre] == g[pre] { + pre++ + } + suf := 0 + for suf < len(w)-pre && suf < len(g)-pre && w[len(w)-1-suf] == g[len(g)-1-suf] { + suf++ + } + var b strings.Builder + fmt.Fprintf(&b, "first difference at line %d:\n", pre+1) + write := func(sign string, lines []string) { + for i, line := range lines { + if i == maxLines { + fmt.Fprintf(&b, "%s ... (%d more lines)\n", sign, len(lines)-maxLines) + return + } + fmt.Fprintf(&b, "%s %s\n", sign, line) + } + } + write("-", w[pre:len(w)-suf]) + write("+", g[pre:len(g)-suf]) + return b.String() +} diff --git a/cmd/bd/testdata/command_tree/help/bd.golden b/cmd/bd/testdata/command_tree/help/bd.golden new file mode 100644 index 000000000..619e70296 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd.golden @@ -0,0 +1,163 @@ +Issues chained together like beads. A lightweight issue tracker with first-class dependency support. + +Usage: + bd [flags] + bd [command] + +Working With Issues: + assign Assign an issue to someone + children List child beads of a parent + close Close one or more issues + comment Add a comment to an issue + comments View or manage comments on an issue + create Create a new issue (or batch from markdown/graph JSON) + create-form Create a new issue using an interactive form + delete Delete one or more issues and clean up references + edit Edit an issue field in $EDITOR + gate Manage async coordination gates + heartbeat Refresh the lease on an issue you hold in_progress + label Manage issue labels + link Link two issues with a dependency + list List issues + merge-slot Manage merge-slot gates for serialized conflict resolution + note Append a note to an issue + priority Set the priority of an issue + promote Promote a wisp to a permanent bead + provenance Append-only provenance event log + q Quick capture: create issue and output only ID + query Query issues using a simple query language + reclaim Revert stale-lease in_progress issues back to ready (dead-worker recovery) + reopen Reopen one or more closed issues + search Search issues by text query + set-state Set operational state (creates event + updates label) + show Show issue details + state Query the current value of a state dimension + tag Add a label to an issue + todo Manage TODO items (convenience wrapper for task issues) + unclaim Release a claimed issue + update Update one or more issues + +Views & Reports: + count Count issues matching filters + diff Show changes between two commits or branches + find-duplicates Find semantically similar issues using text analysis or AI + history Show version history for an issue + lint Check issues for missing template sections + stale Show stale issues (not updated recently) + status Show issue database overview and statistics + statuses List valid issue statuses + types List valid issue types + versions List the recorded versions of a bead + +Dependencies & Structure: + dep Manage dependencies + duplicate Mark an issue as a duplicate of another + duplicates Find and optionally merge duplicate issues + epic Epic management commands + graph Display issue dependency graph + supersede Mark an issue as superseded by a replacement issue + swarm Swarm management for structured epics + +Sync & Data: + backup Back up your beads database + branch List or create branches + conflicts Inspect and resolve live merge conflicts + export Export issues to JSONL format + federation Manage peer-to-peer federation with other workspaces + import Import issues from a JSONL file or stdin into the database + restore Restore the pre-compaction content of a compacted issue + sync Pull, check for conflicts, repair is_blocked, and push (the federation loop) + vc Version control operations + +Setup & Configuration: + bootstrap Non-destructive database setup for fresh clones and recovery + config Manage configuration settings + connect Attach this workspace to a remote bd serve over HTTP + context Show effective backend identity and repository context + dolt Configure Dolt database settings + forget Remove a persistent memory + hooks Manage git hooks for beads integration + human Show essential commands for human users + info Show database information + init Initialize bd in the current directory + kv Key-value store commands + memories List or search persistent memories + migrate-personal Move personal planning issues from the project database to your planning repo + onboard Display minimal snippet for agent instructions file + prime Output AI-optimized workflow context + quickstart Quick start guide for bd + recall Retrieve a specific memory + remember Store a persistent memory + setup Setup integration with AI editors + where Show active beads location + +Maintenance: + batch Run multiple write operations in a single database transaction + compact Squash old Dolt commits to reduce history size + doctor Check and fix beads installation health (start here) + events Read and manage the durable events journal + flatten Squash all Dolt history into a single commit + gc Garbage collect: decay old issues, compact Dolt commits, run Dolt GC + migrate Database migration commands + ping Check database connectivity + preflight Show PR readiness checklist + prune Delete old closed beads to reclaim space and shrink exports + purge Delete closed ephemeral beads to reclaim space + recompute-blocked Recompute is_blocked for all issues (repairs stale flags after a pull) + rename-prefix Rename the issue prefix for all issues in the database + rules Audit and compact Claude rules + sql Execute raw SQL against the beads database + upgrade Check and manage bd version upgrades + worktree Manage git worktrees for parallel development + +Integrations & Advanced: + admin Administrative commands for database maintenance + jira Jira integration commands + linear Linear integration commands + repo Manage multiple repository configuration + +Additional Commands: + ado Azure DevOps integration commands + audit Record and label agent interactions (append-only JSONL) + blocked Show blocked issues + completion Generate the autocompletion script for the specified shell + cook Compile a formula into a proto (ephemeral by default) + defer Defer one or more issues for later + formula Manage workflow formulas + github GitHub integration commands + gitlab GitLab integration commands + help Help about any command + init-safety Explain bd init flag semantics and the destroy-token format + mail Delegate to mail provider (e.g., gt mail) + metrics Show or change anonymous usage-metrics settings + mol Molecule commands (work templates) + notion Notion integration commands + orphans Identify orphaned issues (referenced in commits but still open) + ready Show ready work (open, no active blockers) + rename Rename an issue ID + schema Print the JSON Schema for bd's --json / export output + serve Serve the beads HTTP API over loopback + ship Publish a capability for cross-project dependencies + undefer Undefer one or more issues (restore to open) + version Print version information + +Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + -h, --help help for bd + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output + -V, --version Print version information + +Use "bd [command] --help" for more information about a command. diff --git a/cmd/bd/testdata/command_tree/help/bd_close.golden b/cmd/bd/testdata/command_tree/help/bd_close.golden new file mode 100644 index 000000000..eefb0db30 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_close.golden @@ -0,0 +1,57 @@ +Close one or more issues. + +If no issue ID is provided, closes the last touched issue (from most recent +create, update, show, or close operation). This fallback only applies in +interactive sessions (stdin is a terminal); in scripts and agent sessions a +missing ID is an error, so a command built from an empty variable cannot +silently close an unrelated issue. Set BD_LAST_TOUCHED_FALLBACK=1 to allow +the fallback anywhere, or =0 to disable it entirely. + +When closing multiple issues, provide one --reason for all IDs or repeat +--reason once per ID. Reasons map positionally: the first --reason applies +to the first ID, the second --reason to the second ID, regardless of where +the flags appear in the command line. + +Closing a molecule's last open step also closes the molecule's root, with +the reason "all steps complete", when the root is a poured molecule, a wisp, +or an epic labeled "template"; any other epic stays open as close-eligible +work. A root that would refuse its own close (an open blocker, a pin, another +assignee) stays open with a warning, and the step's close stands. Under +--json, the step's entry names the closed root (auto_closed_molecule) or the +refusal (molecule_auto_close_refusal). --continue claims the next ready step +in the molecule for you; add --no-auto to only show it. + +Usage: + bd close [id...] [flags] + +Aliases: + close, done + +Flags: + --claim-next Automatically claim the next highest priority available issue + --continue Auto-advance to next step in molecule + -f, --force Force close pinned issues or unsatisfied gates + -h, --help help for close + --if-revision bd show --json Apply the write only if the issue's current revision equals this value (the decimal int64 from bd show --json's "revision" field). One id only: refused before any write when combined with more than one issue id. A mismatch writes nothing and exits 13 (vs 1 for other failures). + --no-auto With --continue, show next step but don't claim it + -r, --reason string Reason for closing + --reason-file string Read close reason from file (use - for stdin) + --session string Claude Code session ID (or set CLAUDE_SESSION_ID env var) + --suggest-next Show newly unblocked issues after closing + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_config.golden b/cmd/bd/testdata/command_tree/help/bd_config.golden new file mode 100644 index 000000000..a5bed0d60 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_config.golden @@ -0,0 +1,122 @@ +Manage configuration settings for external integrations and preferences. + +Configuration is stored per-project in the beads database and is version-control-friendly. + +Common namespaces: + - export.* Auto-export settings (stored in config.yaml) + - import.* JSONL import settings (stored in config.yaml) + - jira.* Jira integration settings + - linear.* Linear integration settings + - github.* GitHub integration settings + - gitlab.* GitLab integration settings + - ado.* Azure DevOps integration settings + - notion.* Notion integration settings + - custom.* Custom integration settings + - status.* Issue status configuration + - claim.* Claim arbitration settings (pool-aware claiming) + - doctor.suppress.* Suppress specific bd doctor warnings (GH#1095) + - lint.* Additional per-type lint sections (stored in config.yaml) + +Auto-Export (config.yaml): + Optional JSONL export to .beads/issues.jsonl after write commands (throttled). + Useful for viewers (bv), interchange, and issue-level migration; not a backup. + It is not cross-machine sync; use bd dolt push/pull with a Dolt remote. + Disabled by default. Enable only for integrations that need fresh JSONL. + Auto-staging is separate and disabled by default. + + Keys: + export.auto Enable/disable auto-export (default: false) + export.path Output filename relative to .beads/ (default: issues.jsonl) + export.interval Minimum time between exports (default: 60s) + export.git-add Auto-stage the export file (default: false) + +Auto-Import (config.yaml): + Reads .beads/issues.jsonl by default when a JSONL import path is implied. + Use a relative filename/path so the import stays within the project .beads/ + directory and remains portable across machines. + + Keys: + import.path Input filename relative to .beads/ (default: issues.jsonl) + +Custom Status States: + You can define custom status states for multi-step pipelines using the + status.custom config key. Statuses should be comma-separated. + + Example: + bd config set status.custom "awaiting_review,awaiting_testing,awaiting_docs" + + This enables issues to use statuses like 'awaiting_review' in addition to + the built-in statuses (open, in_progress, blocked, deferred, closed). + +Claim Pools: + A dispatcher can pre-assign issues to a pool pseudo-assignee (e.g. + "fable-crew") and let any actor take them with --claim. List the pool + aliases in the claim.pools config key, comma-separated: + + bd config set claim.pools "fable-crew,night-crew" + + Issues assigned to a real actor (or to an alias not in the list) keep + their anti-steal protection. Pool takes carry the normal lease; note + that if a taker's lease expires, bd reclaim returns the issue to the + unassigned pool, not to the pool alias it was dispatched to. + +Suppressing Doctor Warnings: + Suppress specific bd doctor warnings by check name slug: + bd config set doctor.suppress.pending-migrations true + bd config set doctor.suppress.git-hooks true + Check names are converted to slugs: "Git Hooks" → "git-hooks". + Only warnings are suppressed (errors and passing checks always show). + To unsuppress: bd config unset doctor.suppress. + +Examples: + bd config set export.auto true # Enable auto-export for viewer integrations + bd config set export.path "beads.jsonl" # Custom export filename + bd config set import.path "beads.jsonl" # Custom import filename + bd config set export.git-add true # Also stage the export file + bd config set jira.url "https://company.atlassian.net" + bd config set jira.project "PROJ" + bd config set status.custom "awaiting_review,awaiting_testing" + bd config set claim.pools "fable-crew,night-crew" # Pool aliases claimable by any actor + bd config set doctor.suppress.pending-migrations true + bd config set dolt.debug true # Enable Dolt sql-server debug mode (loglevel=debug, --prof cpu) + bd config set dolt.local-only true # Skip wiring a Dolt sync remote during bd init + bd config set lint.sections.epic "Standards scorecard, Cost" # Extra sections bd lint requires for epics + bd config get export.auto + bd config list + bd config unset jira.url + +Usage: + bd config [command] + +Available Commands: + apply Reconcile system state to match configuration + drift Detect config-vs-reality inconsistencies + get Get a configuration value + list List all configuration + set Set a configuration value + set-many Set multiple configuration values in one operation + show Show all effective configuration with provenance + unset Delete a configuration value + validate Validate sync-related configuration + +Flags: + -h, --help help for config + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output + +Use "bd config [command] --help" for more information about a command. diff --git a/cmd/bd/testdata/command_tree/help/bd_create.golden b/cmd/bd/testdata/command_tree/help/bd_create.golden new file mode 100644 index 000000000..adc90d852 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_create.golden @@ -0,0 +1,72 @@ +Create a new issue (or batch from markdown/graph JSON) + +Usage: + bd create [title] [flags] + +Aliases: + create, new + +Flags: + --acceptance string Acceptance criteria + --allow-empty-description Allow empty description input from stdin or file + --append-notes string Append to existing notes (with newline separator) + -a, --assignee string Assignee + --body-file string Read description from file (use - for stdin) + --context string Additional context for the issue + --defer string Defer until date (issue hidden from bd ready until then). Same formats as --due + --deps strings Dependencies as 'type:id' or bare 'id'. Bare 'id', 'depends-on:id', and 'blocked-by:id' all make THIS issue depend on id; 'blocks:id' reverses direction (id depends on this issue). E.g. 'blocked-by:bd-20,discovered-from:bd-15' + -d, --description string Issue description + --design string Design notes + --design-file string Read design from file (use - for stdin) + --dry-run Preview what would be created without actually creating + --due string Due date/time. Formats: +6h, +1d, +2w, tomorrow, next monday, 2025-01-15 + --ephemeral Create as ephemeral (short-lived, subject to TTL compaction) + -e, --estimate int Time estimate in minutes (e.g., 60 for 1 hour) + --event-actor string Entity URI who caused this event (requires --type=event) + --event-category string Event category (e.g., patrol.muted, agent.started) (requires --type=event) + --event-payload string Event-specific JSON data (requires --type=event) + --event-target string Entity URI or bead ID affected (requires --type=event) + --external-ref string External reference (e.g., 'gh-9', 'jira-ABC', Linear URL) + -f, --file string Create one issue per ## heading (### fields attach to that issue); for title plus description-from-file, use --body-file + --force Force creation even if prefix doesn't match database prefix + --graph string Create a graph of issues with dependencies from JSON plan file + -h, --help help for create + --id string Explicit issue ID (e.g., 'bd-42' for partitioning) + -l, --labels strings Labels (comma-separated) + --metadata string Set custom metadata (JSON object, or @file.json to read from file) + --mol-type string Molecule type: swarm (multi-agent), patrol (recurring ops), work (default) + --no-history Skip Dolt commit history without making GC-eligible (for permanent agent beads) + --no-inherit-labels Don't inherit labels from parent issue + --notes string Notes + --parent string Parent issue ID for hierarchical child (e.g., 'bd-a3f8e9') + -p, --priority string Priority (0-4 or P0-P4, 0=highest) (default "2") + --repo string Target repository for issue (overrides auto-routing) + --silent Output only the issue ID (for scripting) + --skills string Required skills for this issue + --spec-id string Link to specification document + -s, --status string Initial status + --stdin Read description from stdin (alias for --body-file -) + --storage-class string Storage class: versioned, unversioned, or ephemeral (default: storage-class. config, else versioned) + --title string Issue title (alternative to positional argument) + -t, --type string Issue type (bug|feature|task|epic|chore|decision|spike|story|milestone); custom types require types.custom config; aliases: enhancement/feat→feature, dec/adr→decision (default "task") + --validate Validate description contains required sections for issue type + --waits-for string Spawner issue ID to wait for (creates waits-for dependency for fanout gate) + --waits-for-gate string Gate type: all-children (wait for all) or any-children (wait for first) (default "all-children") + --wisp-type string Wisp type for TTL-based compaction: heartbeat, ping, patrol, gc_report, recovery, error, escalation + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_list.golden b/cmd/bd/testdata/command_tree/help/bd_list.golden new file mode 100644 index 000000000..d9c582c3a --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_list.golden @@ -0,0 +1,89 @@ +List issues + +Usage: + bd list [flags] + +Flags: + --all Show all issues including closed (overrides default filter) + -a, --assignee string Filter by assignee + --brief Omit the free-form text (description, design, acceptance criteria, notes, payload, waiters) from each row. Filters that read those fields, such as --desc-contains, still select on them. An omitted field is indistinguishable from an empty one in --json; fetch a whole issue with bd show. + --closed-after string Filter issues closed after date (YYYY-MM-DD or RFC3339) + --closed-before string Filter issues closed before date (YYYY-MM-DD or RFC3339) + --created-after string Filter issues created after date (YYYY-MM-DD or RFC3339) + --created-before string Filter issues created before date (YYYY-MM-DD or RFC3339) + --defer-after string Filter issues deferred after date (supports relative: +6h, tomorrow) + --defer-before string Filter issues deferred before date (supports relative: +6h, tomorrow) + --deferred Show only issues with defer_until set + --deps string[="scheduling"] Annotate tree with dependency edges and order siblings by them: 'scheduling' (bare --deps) or 'all' + --desc-contains string Filter by description substring (case-insensitive) + --due-after string Filter issues due after date (supports relative: +6h, tomorrow) + --due-before string Filter issues due before date (supports relative: +6h, tomorrow) + --empty-description Filter issues with empty or missing description + --exclude-label strings Exclude issues that have ANY of these labels + --exclude-type strings Exclude issue types from results (comma-separated or repeatable, e.g., --exclude-type=convoy,epic) + --external-contains string Filter by external ref substring (case-insensitive) + --external-ref string Filter by exact external_ref value + --flat Disable tree format and use legacy flat list output + --format string Export the dependency graph of the listed issues: 'digraph' (edge list for golang.org/x/tools/cmd/digraph), 'dot' (Graphviz), or a Go template rendered once per edge over .IssueID, .DependsOnID, .Type, .Issue, .Dependency. 'json' is the exception and does not export the graph: it is equivalent to --json, which also wins when both flags are given. 'digraph' and templates print nothing when no listed issue depends on another; 'dot' still prints the nodes + --has-metadata-key string Filter issues that have this metadata key set + -h, --help help for list + --id string Filter by specific issue IDs (comma-separated or repeated, e.g., bd-1,bd-5,bd-10) + --include-comments Populate each row's comments field with full comment bodies in JSON output (--json only; costs one query per row that has comments). Without it, a row with comments carries comments_omitted=true, so an absent comments field is never mistaken for having none. Text output is unaffected either way. + --include-ephemeral Include ephemeral wisp-plane rows in output (normally hidden) + --include-gates Include gate issues in output (normally hidden) + --include-infra Include infrastructure beads (agent/role/message) in output + --include-templates Include template molecules in output + -l, --label strings Filter by labels (AND: must have ALL). Can combine with --label-any + --label-any strings Filter by labels (OR: must have AT LEAST ONE). Can combine with --label + --label-pattern string Filter by label glob pattern (e.g., 'tech-*' matches tech-debt, tech-legacy) + --label-regex string Filter by label regex pattern (e.g., 'tech-(debt|legacy)') + -n, --limit int Limit results (an explicit --limit always wins, 0 meaning unlimited; otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50) (default 50) + --long Show detailed multi-line output for each issue + --max-rows int Hard upper bound on rows returned. Returns a non-zero exit (code 2) and an error to stderr if exceeded. 0 disables (the default). Overrides BEADS_MAX_ROWS for this invocation. Useful in CI/agent rigs that want a circuit breaker against pathological queries. Honored on both the direct and the --proxied-server route. + --metadata-field stringArray Filter by metadata field (key=value, repeatable) + --mol-type string Filter by molecule type: swarm, patrol, or work + --no-assignee Filter issues with no assignee + --no-labels Filter issues with no labels + --no-pager Disable pager output + --no-parent Exclude child issues (show only top-level issues) + --no-pinned Exclude pinned issues + --notes-contains string Filter by notes substring (case-insensitive) + --offset int Skip the first N matching results (0-based). Only supported under --proxied-server. + --overdue Show only issues with due_at in the past (not closed) + --parent string Filter by parent issue ID (shows children of specified issue) + --pinned Show only pinned issues + --pretty Display issues in a tree format with status/priority symbols + -p, --priority string Priority (0-4 or P0-P4, 0=highest) + --priority-max string Filter by maximum priority (inclusive, 0-4 or P0-P4) + --priority-min string Filter by minimum priority (inclusive, 0-4 or P0-P4) + --ready Show only ready issues (no active blockers, same semantics as bd ready) + -r, --reverse Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first) + --skip-labels Skip label hydration. The labels field in output will be empty regardless of actual labels. Use only when the caller does not depend on label data. Cannot combine with --label, --label-any, --label-pattern, --label-regex, --exclude-label, or --no-labels. + --sort string Sort by field: priority, created, updated, closed, status, id, title, type, assignee + --spec string Filter by spec_id prefix + -s, --status string Filter by stored status (open, in_progress, blocked, deferred, closed). Comma-separated or repeated for multiple: --status open,in_progress + --title string Filter by title text (case-insensitive substring match) + --title-contains string Filter by title substring (case-insensitive) + --tree Hierarchical tree format (default: true; use --flat to disable) (default true) + -t, --type string Filter by type (bug, feature, task, epic, chore, decision, merge-request, molecule, gate, convoy). Aliases: mr→merge-request, feat→feature, mol→molecule, dec/adr→decision + --updated-after string Filter issues updated after date (YYYY-MM-DD or RFC3339) + --updated-before string Filter issues updated before date (YYYY-MM-DD or RFC3339) + -w, --watch Watch for changes and auto-update display (implies --pretty) + --wisp-type string Filter by wisp type: heartbeat, ping, patrol, gc_report, recovery, error, escalation + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_serve.golden b/cmd/bd/testdata/command_tree/help/bd_serve.golden new file mode 100644 index 000000000..848d00820 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_serve.golden @@ -0,0 +1,119 @@ +Serve the beads HTTP API — the same work surface the CLI answers, for +automation clients that would otherwise fork a bd subprocess per call. + +The wire contract is described by an OpenAPI document (/v0); GET +/v0/beads/context reports which operations this build actually implements. + +DEPLOYMENT + + Pass an explicit port. The default 127.0.0.1:0 takes an ephemeral one, which + is right for ad-hoc and test use — where the bound address printed on stdout + is read immediately — but carries no mutual exclusion: two serves against one + workspace then run side by side on different ports with no way to enumerate + them. On a fixed port the second one fails to bind, which is the intended + behavior. Concurrent serves are data-safe either way; claims are arbitrated + in the SQL server. + + Run it under a supervisor. bd shuts down gracefully on SIGHUP as well as + SIGINT and SIGTERM, so closing the terminal of a foreground bd serve stops it. + +PROBES + + GET /healthz is LIVENESS only: it answers from the process and never touches + the database, so it stays green while the database is unreachable. For + readiness use GET /v0/beads/ready?limit=1 — a real query, where 200 means + ready and 503 means live but not ready. + +AUTHENTICATION + + Optional, and off by default on loopback — where the trust model is the + loopback boundary itself, the same one the database behind it already relies + on. --auth-token-file turns it on: every operation except GET /healthz then + requires an "Authorization: Bearer " header, GET /v0/beads/context + included, because it reports the repo root, beads directory and database name. + + The file holds ONE TOKEN PER LINE and every line is accepted. That is the + rotation mechanism: write the new token alongside the old, roll the clients + over, then delete the old line. The file is re-read while the server runs, so + both the addition and the removal take effect within about a second and + neither needs a restart. Write it atomically (a temp file plus rename; a + Kubernetes secret mount already does this). + + There is deliberately no --auth-token flag. A credential passed as an + argument is readable by every local user in the process listing. + + --allow-non-loopback REQUIRES a token file. Beyond loopback, reaching the + address would otherwise be the whole authorization: any peer could read every + issue and claim work as any actor. --insecure-no-auth is the explicit, + auditable way to say you meant that anyway. + + The Host allowlist is what a service deployment usually trips over first. The + DNS-rebinding check answers only to loopback spellings and the bind address, + so a client dialing a service name gets 400 on every request; enumerate the + names it dials with --allowed-host (repeatable). Matching is exact — no + wildcards — and the startup log line prints the effective allowlist. + +WHAT THIS DOES NOT DO + + No TLS. Even with a token, the credential and every issue body travel in + plaintext, so a deployment beyond loopback has to supply confidentiality + itself — a service mesh, or a network boundary you already trust. + + Hooks do not fire. A hook is a user-controlled subprocess per mutation: in a + concurrent server that is an unbounded latency multiplier and an orphaned + child at shutdown, and its working-directory-derived hook lookup is + meaningless in a server process. A CLI claim runs on_update; an HTTP claim + does not. + + The per-command auto-commit machinery does not run. Durability is per request: + a successful claim commits inside its own transaction, exactly as a proxied + CLI claim does today. + + An actor on an HTTP request is caller-asserted provenance for the audit trail, + not authenticated identity — the same thing it has always been on the CLI, + where any local process can pass any --actor. + + It does not run under --readonly, and refuses to start rather than binding. + Every server it binds publishes the issue-claim operation, and the capability + set it advertises is a property of the build rather than of the flags on the + process that started it — so a read-only server would advertise a write it + could never land. + +DESTRUCTIVE OPERATIONS + + POST /v0/beads/issues:sweep deletes closed beads in bulk — the operation + behind bd purge and bd prune — and nothing it deletes comes back. It shares + the library surface those commands call, so it inherits their guards: pinned + beads are never swept, and a durable sweep with neither a cutoff nor an id + pattern is refused rather than clearing every closed bead in the workspace. + Combined with the trust model above, that means anyone who can reach this + address can erase closed work; bind it accordingly. + +Usage: + bd serve [flags] + +Flags: + --addr string Address to bind as IP:PORT; the host must be a numeric IP literal, and port 0 takes an ephemeral port (default "127.0.0.1:0") + --allow-non-loopback Permit a bind beyond loopback. Requires --auth-token-file, since reaching the address would otherwise be the whole authorization + --allowed-host stringArray Additional Host header value to answer to, e.g. a service DNS name. Repeatable; matched exactly, with no wildcards + --auth-token-file string Require an Authorization: Bearer token from this file, one token per line, all accepted. Re-read while running, so rewriting it rotates tokens without a restart (env BEADS_SERVE_TOKEN_FILE) + -h, --help help for serve + --insecure-no-auth Serve a non-loopback bind with NO authentication. Every peer that can reach the address gets full read and claim access + --large-apply-ceiling duration Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long plus 5s, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md (default 5m0s) + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_show.golden b/cmd/bd/testdata/command_tree/help/bd_show.golden new file mode 100644 index 000000000..595a7dd74 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_show.golden @@ -0,0 +1,41 @@ +Show issue details + +Usage: + bd show [id...] [--id=...] [--current] [flags] + +Aliases: + show, view + +Flags: + --as-of string Show issue as it existed at a specific commit hash or branch (requires Dolt; commit hashes must be the full 32-character hash from 'bd history', not a prefix) + --brief-deps Reduce each dependency to its identity fields in JSON output (--json only; drops description, design, notes and acceptance criteria) + --children Show only the children of this issue + --comments-tail int Render only the last N comments in text output, preceded by one elision line naming how many older ones were hidden (text output only; 0, the default, renders every comment unchanged; N must be >= 0) + --current Show the currently active issue (in-progress, hooked, or last touched) + -h, --help help for show + --id stringArray Issue ID (use for IDs that look like flags, e.g., --id=gt--xyz) + --include-comments Stream full comment bodies in JSON output (--json only; may be slow on issues with many comments). Without it the comments field is absent entirely, not truncated: an issue with comments carries comment_count and comments_omitted=true, so check those before reading an absent comments field as none. + --include-dependents Stream full dependent issues in JSON output (--json only; may be slow on hub beads) + --local-time Show timestamps in local time instead of UTC (date-only fields are always local) + --long Show all available fields (extended metadata, agent identity, gate fields, etc.) + --refs Show issues that reference this issue (reverse lookup) + --short Show compact one-line output per issue + --thread Show full conversation thread (for messages) + -w, --watch Watch for changes and auto-refresh display + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_update.golden b/cmd/bd/testdata/command_tree/help/bd_update.golden new file mode 100644 index 000000000..37451cdc6 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_update.golden @@ -0,0 +1,78 @@ +Update one or more issues. + +If no issue ID is provided, updates the last touched issue (from most recent +create, update, show, or close operation). This fallback only applies in +interactive sessions (stdin is a terminal); in scripts and agent sessions a +missing ID is an error, so a command built from an empty variable cannot +silently mutate an unrelated issue. Set BD_LAST_TOUCHED_FALLBACK=1 to allow +the fallback anywhere, or =0 to disable it entirely. + +Updates are applied per issue ID, not atomically across IDs: when some IDs +fail, the remaining issues are still updated, every failed ID is reported on +stderr, and the command exits nonzero. + +Exit codes: 1 for general failures; 13 when every failure is a stale +--if-assignee/--if-status guard (the precondition no longer held, nothing was +written — another actor won the race, so retrying the same guard is +pointless). + +Usage: + bd update [id...] [flags] + +Flags: + --acceptance string Acceptance criteria + -l, --add-label strings Add labels (repeatable); -l matches bd create -l + --allow-empty-description Allow empty description replacement when reading from stdin or file + --append-notes string Append to existing notes (with newline separator) + -a, --assignee string Assignee + --await-id string Set gate await_id (e.g., GitHub run ID for gh:run gates) + --body-file string Read description from file (use - for stdin) + --claim Atomically claim the issue (sets assignee to you, status to in_progress; idempotent if already claimed by you; issues assigned to a pool alias listed in the claim.pools config are claimable too) + --clear-notes Clear the notes field (--notes "" is refused: an empty value is usually a dead command substitution) + --defer string Defer until date (empty to clear). Issue hidden from bd ready until then, then auto-wakes to open + -d, --description string Issue description + --design string Design notes + --design-file string Read design from file (use - for stdin) + --due string Due date/time (empty to clear). Formats: +6h, +1d, +2w, tomorrow, next monday, 2025-01-15 + --ephemeral Mark issue as ephemeral (wisp) - not exported to JSONL + -e, --estimate int Time estimate in minutes (e.g., 60 for 1 hour) + --external-ref string External reference (e.g., 'gh-9', 'jira-ABC', Linear URL) + --force Override refusals: let -a/--assignee overwrite another actor's live in_progress claim (use only for abandoned claims — crashed agent, expired lease; prefer bd reclaim), let -s/--status move the issue into closed (or a configured done status) despite open children or a live blocker (same as bd close --force), and let --notes overwrite existing non-empty notes (use --append-notes to preserve history instead) + -h, --help help for update + --history Clear no-history flag (re-enable Dolt commit history) + --if-assignee string Apply the update only if the current assignee equals this value (--if-assignee '' requires unassigned); a mismatch writes nothing and exits 13 (vs 1 for other failures). Requires a field update; cannot combine with --claim + --if-revision bd show --json Apply the write only if the issue's current revision equals this value (the decimal int64 from bd show --json's "revision" field). One id only: refused before any write when combined with more than one issue id. A mismatch writes nothing and exits 13 (vs 1 for other failures). Composes with --if-assignee/--if-status: all guards present must hold. + --if-status string Apply the update only if the current status equals this value; a mismatch writes nothing and exits 13 (vs 1 for other failures). Requires a field update; cannot combine with --claim + --metadata string Set custom metadata (JSON object, or @file.json to read from file) + --no-history Mark issue as no-history (skip Dolt commits, not GC-eligible) + --notes string Replace the notes field (requires --force over existing non-empty notes; --clear-notes clears; --append-notes appends instead) + --parent string New parent issue ID (reparents the issue, use empty string to remove parent) + --persistent Mark issue as persistent (promote wisp to regular issue) + -p, --priority string Priority (0-4 or P0-P4, 0=highest) + --remove-label strings Remove labels (repeatable) + --session string Claude Code session ID for status=closed (or set CLAUDE_SESSION_ID env var) + --set-labels strings Set labels, replacing all existing (repeatable) + --set-metadata stringArray Set metadata key=value (repeatable, e.g., --set-metadata team=platform) + --spec-id string Link to specification document + -s, --status string New status + --stdin Read description from stdin (alias for --body-file -) + --title string New title + -t, --type string New type (bug|feature|task|epic|chore|decision|spike|story|milestone); custom types require types.custom config; aliases: enhancement/feat→feature, dec/adr→decision + --unset-metadata stringArray Remove metadata key (repeatable, e.g., --unset-metadata team) + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_version.golden b/cmd/bd/testdata/command_tree/help/bd_version.golden new file mode 100644 index 000000000..cb5978ad7 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_version.golden @@ -0,0 +1,24 @@ +Print version information + +Usage: + bd version [flags] + +Flags: + -h, --help help for version + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/help/bd_versions.golden b/cmd/bd/testdata/command_tree/help/bd_versions.golden new file mode 100644 index 000000000..54d21dac5 --- /dev/null +++ b/cmd/bd/testdata/command_tree/help/bd_versions.golden @@ -0,0 +1,60 @@ +List the versions recorded for a bead by versioned history. + +Recording is off by default. Turn it on with: + + bd config set versioned-history.enabled true + +That writes the setting into the store, so every client of that store agrees +about whether a write is recorded, and it is the setting this command reports. +BD_VERSIONED_HISTORY_ENABLED=1 (or a value in config.yaml) also turns recording +on for the writes a process makes, but it says nothing about the store, and +this command only reads, so it does not consult them. + +Either source turning it on is enough for a writer; neither can switch the +other off. The consequence worth knowing: once the store setting says on, +BD_VERSIONED_HISTORY_ENABLED=0 will not turn recording off for one run. Turn +it off where it was turned on: + + bd config set versioned-history.enabled false + +Only beads created while recording is on, or that enter the issues plane +while it is on (a wisp promoted or made persistent), are versioned. A bead +already on the issues plane when recording was turned on is legacy: editing +it records nothing (the write fence, design §16.2b), and nothing is +backfilled. + +Turning recording off does not hide what was already recorded — this command +still lists it, and says that recording is currently off. + +A caution while the feature is young: keep recording ON for every writer of a +shared store, or leave it off for all of them. A write from a client with +recording off advances nothing, and the next recorded write mints a version +that absorbs that unrecorded change under its own actor — the revisions stay +contiguous and the listing looks gapless when it is not. + +Examples: + bd versions bd-123 + bd versions bd-123 --json + +Usage: + bd versions [flags] + +Flags: + -h, --help help for versions + +Global Flags: + --actor string Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER) + --cpu-profile Generate CPU profile for performance analysis + --database string Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only) + --db string Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database) + -C, --directory string Change to this directory before running the command (like git -C) + --dolt-auto-commit string Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit + --global Use the global shared-server database (beads_global) + --ignore-schema-skew Proceed despite forward schema drift (some queries may fail) + --json Output in JSON format + --mem-profile string Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE) + --no-color Disable color output (also: NO_COLOR=1 or CLICOLOR=0) + -q, --quiet Suppress non-essential output (errors only) + --readonly Read-only mode: block write operations (for worker sandboxes) + --sandbox Sandbox mode: disables Dolt auto-push + -v, --verbose Enable verbose/debug output diff --git a/cmd/bd/testdata/command_tree/tree.golden b/cmd/bd/testdata/command_tree/tree.golden new file mode 100644 index 000000000..79d89ce5e --- /dev/null +++ b/cmd/bd/testdata/command_tree/tree.golden @@ -0,0 +1,7461 @@ +# bd command tree (cgo build), pinned by TestCommandTreeGolden. +# Regenerate after an intentional CLI change: +# BD_UPDATE_GOLDEN=1 CGO_ENABLED=1 go test -tags gms_pure_go ./cmd/bd -run '^TestCommand(Tree|Help)Golden$' + +bd + use: "bd" + short: "bd - Dependency-aware issue tracker" + hidden: false + runnable: true + long: + | Issues chained together like beads. A lightweight issue tracker with first-class dependency support. + persistent-flag --actor type=string default="" usage="Actor name for audit trail (default: $BEADS_ACTOR, git user.name, $USER)" + persistent-flag --cpu-profile type=bool default="false" usage="Generate CPU profile for performance analysis" + persistent-flag --database type=string default="" usage="Run against a different server database for this invocation, without changing the project's configured database (proxied-server mode only)" + persistent-flag --db type=string default="" usage="Database path (default: auto-discover .beads/*.db). In proxied-server mode, a value that isn't an existing path is treated as a database name override (see --database)" + persistent-flag --directory -C type=string default="" usage="Change to this directory before running the command (like git -C)" + persistent-flag --dolt-auto-commit type=string default="" usage="Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit" + persistent-flag --format type=string default="" hidden usage="Output format (json). Alias for --json" + persistent-flag --global type=bool default="false" usage="Use the global shared-server database (beads_global)" + persistent-flag --ignore-schema-skew type=bool default="false" usage="Proceed despite forward schema drift (some queries may fail)" + persistent-flag --json type=bool default="false" usage="Output in JSON format" + persistent-flag --mem-profile type=string default="" usage="Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE)" + persistent-flag --no-color type=bool default="false" usage="Disable color output (also: NO_COLOR=1 or CLICOLOR=0)" + persistent-flag --quiet -q type=bool default="false" usage="Suppress non-essential output (errors only)" + persistent-flag --readonly type=bool default="false" usage="Read-only mode: block write operations (for worker sandboxes)" + persistent-flag --sandbox type=bool default="false" usage="Sandbox mode: disables Dolt auto-push" + persistent-flag --verbose -v type=bool default="false" usage="Enable verbose/debug output" + flag --version -V type=bool default="false" usage="Print version information" + +bd admin + use: "admin" + short: "Administrative commands for database maintenance" + group: "advanced" + hidden: false + runnable: false + long: + | Administrative commands for beads database maintenance. + | + | These commands are for advanced users and should be used carefully: + | cleanup Delete closed issues (issue lifecycle) + | compact Compact old closed issues to save space (storage optimization) + | reset Remove all beads data and configuration (full reset) + | + | For routine maintenance, prefer 'bd doctor --fix' which handles common repairs + | automatically. Use these admin commands for targeted database operations. + +bd admin cleanup + use: "cleanup" + short: "Delete closed issues to reduce database size" + hidden: false + runnable: true + long: + | Delete closed issues to reduce database size. + | + | This command permanently removes closed issues from the database. + | + | NOTE: This command only manages issue lifecycle (closed -> deleted). For general + | health checks and automatic repairs, use 'bd doctor --fix' instead. + | + | By default, deletes ALL closed issues. Use --older-than to only delete + | issues closed before a certain date. + | + | EXAMPLES: + | bd admin cleanup --force # Delete all closed issues + | bd admin cleanup --older-than 30 --force # Only issues closed 30+ days ago + | bd admin cleanup --ephemeral --force # Only closed wisps (transient molecules) + | bd admin cleanup --dry-run # Preview what would be deleted + | + | SAFETY: + | - Requires --force flag to actually delete (unless --dry-run) + | - Supports --cascade to delete dependents + | - Shows preview of what will be deleted + | - Use --json for programmatic output + | + | SEE ALSO: + | bd doctor --fix Automatic health checks and repairs (recommended for routine maintenance) + | bd admin compact Compact old closed issues to save space + flag --cascade type=bool default="false" usage="Recursively delete all dependent issues" + flag --dry-run type=bool default="false" usage="Preview what would be deleted without making changes" + flag --ephemeral type=bool default="false" usage="Only delete closed wisps (transient molecules)" + flag --force -f type=bool default="false" usage="Actually delete (without this flag, shows error)" + flag --older-than type=int default="0" usage="Only delete issues closed more than N days ago (0 = all closed issues)" + +bd admin compact + use: "compact" + short: "Compact old closed issues to save space" + hidden: false + runnable: true + long: + | Compact old closed issues using semantic summarization. + | + | Compaction reduces database size by summarizing closed issues that are no longer + | actively referenced. This is permanent graceful decay - original content is discarded. + | + | Modes: + | - Analyze: Export candidates for agent review (no API key needed) + | - Apply: Accept agent-provided summary (no API key needed) + | - Auto: AI-powered compaction (requires ANTHROPIC_API_KEY, MINIMAX_API_KEY, or ai.api_key) + | - Dolt: Run Dolt garbage collection (for Dolt-backend repositories) + | + | Tiers: + | - Tier 1: Semantic compression (30 days closed, 70% reduction) + | - Tier 2: Ultra compression (90 days closed) - planned, not yet implemented + | + | Dolt Garbage Collection: + | With auto-commit per mutation, Dolt commit history grows over time. Use + | --dolt to run Dolt garbage collection and reclaim disk space. + | + | --dolt: Run Dolt GC on the active database to free disk space. + | This removes unreachable commits and compacts storage. + | + | Examples: + | # Dolt garbage collection + | bd compact --dolt # Run Dolt GC + | bd compact --dolt --dry-run # Preview without running GC + | + | # Agent-driven workflow (recommended) + | bd compact --analyze --json # Get candidates with full content + | bd compact --apply --id bd-42 --summary summary.txt + | bd compact --apply --id bd-42 --summary - < summary.txt + | + | # AI-powered workflow + | bd compact --auto --dry-run # Preview candidates + | bd compact --auto --all # Compact all eligible issues + | bd compact --auto --id bd-42 # Compact specific issue + | MINIMAX_API_KEY=... bd compact --auto --all + | + | # Statistics + | bd compact --stats # Show statistics + | + flag --actor type=string default="agent" usage="Actor name for audit trail" + flag --all type=bool default="false" usage="Process all candidates" + flag --analyze type=bool default="false" usage="Analyze mode: export candidates for agent review" + flag --apply type=bool default="false" usage="Apply mode: accept agent-provided summary" + flag --auto type=bool default="false" usage="Auto mode: AI-powered compaction" + flag --batch-size type=int default="10" usage="Issues per batch" + flag --dolt type=bool default="false" usage="Dolt mode: run Dolt garbage collection on the active database" + flag --dry-run type=bool default="false" usage="Preview without compacting" + flag --force type=bool default="false" usage="Force compact (bypass checks, requires --id)" + flag --id type=string default="" usage="Compact specific issue" + flag --limit type=int default="0" usage="Limit number of candidates (0 = no limit)" + flag --stats type=bool default="false" usage="Show compaction statistics" + flag --summary type=string default="" usage="Path to summary file (use '-' for stdin)" + flag --tier type=int default="1" usage="Compaction tier (only tier 1 is implemented)" + flag --workers type=int default="5" usage="Parallel workers" + +bd admin reset + use: "reset" + short: "Remove all beads data and configuration" + hidden: false + runnable: true + long: + | Reset beads to an uninitialized state, removing all local data. + | + | This command removes: + | - The .beads directory (database, JSONL, config) + | - Git hooks installed by bd + | - Sync branch worktrees + | + | By default, shows what would be deleted (dry-run mode). + | Use --force to actually perform the reset. + | + | Examples: + | bd reset # Show what would be deleted + | bd reset --force # Actually delete everything + flag --force type=bool default="false" usage="Actually perform the reset (required)" + +bd ado + use: "ado" + short: "Azure DevOps integration commands" + hidden: false + runnable: false + long: + | Commands for syncing issues between beads and Azure DevOps. + | + | Configuration can be set via 'bd config' or environment variables: + | ado.org / AZURE_DEVOPS_ORG - Organization name + | ado.project / AZURE_DEVOPS_PROJECT - Project name (single) + | ado.projects / AZURE_DEVOPS_PROJECTS - Project names (comma-separated) + | ado.pat / AZURE_DEVOPS_PAT - Personal access token + | ado.url / AZURE_DEVOPS_URL - Custom base URL (on-prem) + +bd ado projects + use: "projects" + short: "List accessible Azure DevOps projects" + hidden: false + runnable: true + long: + | List Azure DevOps projects that the configured token has access to. + +bd ado pull + use: "pull [refs...]" + short: "Pull specific items from Azure DevOps" + hidden: false + runnable: true + long: + | Pull one or more items from Azure DevOps. + | + | Accepts bead IDs or external references as positional arguments. + | Equivalent to: bd ado sync --pull-only --issues + flag --dry-run type=bool default="false" usage="Preview pull without making changes" + +bd ado push + use: "push [bead-ids...]" + short: "Push specific beads to Azure DevOps" + hidden: false + runnable: true + long: + | Push one or more beads issues to Azure DevOps. + | + | Accepts bead IDs as positional arguments. + | Equivalent to: bd ado sync --push-only --issues + flag --dry-run type=bool default="false" usage="Preview push without making changes" + +bd ado status + use: "status" + short: "Show Azure DevOps sync status" + hidden: false + runnable: true + long: + | Display current Azure DevOps configuration and sync status. + +bd ado sync + use: "sync" + short: "Sync issues with Azure DevOps" + hidden: false + runnable: true + long: + | Synchronize issues between beads and Azure DevOps. + | + | By default, performs bidirectional sync: + | - Pulls new/updated work items from Azure DevOps to beads + | - Pushes local beads issues to Azure DevOps + | + | Use --pull-only or --push-only to limit direction. + | + | Filters (--area-path, --iteration-path, --types, --states) restrict + | which work items are synced. On pull, they limit the WIQL query. On push, + | --types and --states filter local beads before pushing to ADO. Use + | --no-create with push to skip creating new ADO work items (only update + | existing linked items). Filters can also be persisted via config: + | ado.filter.area_path, ado.filter.iteration_path, + | ado.filter.types, ado.filter.states + | CLI flags override config values when both are set. + flag --area-path type=string default="" usage="Filter to ADO area path (e.g., \"Project\\Team\")" + flag --bootstrap-match type=bool default="false" usage="Enable heuristic matching for first sync" + flag --dry-run type=bool default="false" usage="Show what would be synced without making changes" + flag --issues type=string default="" usage="Comma-separated bead IDs to sync selectively (e.g., bd-abc,bd-def). Mutually exclusive with --parent." + flag --iteration-path type=string default="" usage="Filter to ADO iteration path (e.g., \"Project\\Sprint 1\")" + flag --no-create type=bool default="false" usage="Never create new items in either direction (pull or push)" + flag --parent type=string default="" usage="Limit push to this bead and its descendants (push only). Mutually exclusive with --issues." + flag --prefer-ado type=bool default="false" usage="On conflict, use Azure DevOps version" + flag --prefer-local type=bool default="false" usage="On conflict, keep local beads version" + flag --prefer-newer type=bool default="false" usage="On conflict, use most recent version (default)" + flag --project type=stringSlice default="[]" usage="Project name(s) to sync (overrides configured project/projects)" + flag --pull-only type=bool default="false" usage="Only pull issues from Azure DevOps" + flag --push-only type=bool default="false" usage="Only push issues to Azure DevOps" + flag --reconcile type=bool default="false" usage="Force reconciliation scan for deleted items" + flag --states type=string default="" usage="Filter to ADO states, comma-separated (e.g., \"New,Active,Resolved\")" + flag --types type=string default="" usage="Filter to work item types, comma-separated (e.g., \"Bug,Task,User Story\")" + +bd assign + use: "assign " + short: "Assign an issue to someone" + group: "issues" + hidden: false + runnable: true + long: + | Assign an issue to someone. + | + | Shorthand for 'bd update --assignee '. + | + | Refuses to overwrite another actor's live in_progress claim without --force + | (bd-98s5c); issues assigned to a claim.pools alias are exempt, matching + | --claim. For a holder-aware transfer prefer + | 'bd update --if-assignee -a '. + | + | Examples: + | bd assign bd-123 alice + | bd assign bd-123 "" # unassign + flag --force type=bool default="false" usage="Allow overwriting another actor's live in_progress claim (use only for abandoned claims — crashed agent, expired lease; prefer bd reclaim)" + flag --if-revision type=string default="" usage="Apply the write only if the issue's current revision equals this value (the decimal int64 from `bd show --json`'s \"revision\" field). One id only: refused before any write when combined with more than one issue id. A mismatch writes nothing and exits 13 (vs 1 for other failures)." + +bd audit + use: "audit" + short: "Record and label agent interactions (append-only JSONL)" + hidden: false + runnable: false + long: + | Record explicit agent/tool interaction audit entries in .beads/interactions.jsonl. + | + | This optional JSONL sidecar is disabled by default. Enable it with: + | + | bd config set audit.enabled true + | + | Issue history is always recorded in the database and is visible with + | bd history --events. The JSONL sidecar is for explicit interaction capture: + | - auditing ("why did the agent do that?") + | - dataset generation (SFT/RL fine-tuning) + | + | Entries are append-only. Labeling creates a new "label" entry that references a parent entry. + +bd audit label + use: "label " + short: "Append a label entry referencing an existing interaction" + hidden: false + runnable: true + flag --label type=string default="" usage="Label value (e.g. \"good\" or \"bad\")" + flag --reason type=string default="" usage="Reason for label" + +bd audit record + use: "record" + short: "Append an audit interaction entry" + hidden: false + runnable: true + flag --error type=string default="" usage="Error string (llm_call/tool_call)" + flag --exit-code type=int default="-1" usage="Exit code (tool_call)" + flag --issue-id type=string default="" usage="Related issue id (bd-...)" + flag --kind type=string default="" usage="Entry kind (e.g. llm_call, tool_call, label)" + flag --model type=string default="" usage="Model name (llm_call)" + flag --prompt type=string default="" usage="Prompt text (llm_call)" + flag --response type=string default="" usage="Response text (llm_call)" + flag --stdin type=bool default="false" usage="Read a JSON object from stdin (must match audit.Entry schema)" + flag --tool-name type=string default="" usage="Tool name (tool_call)" + +bd backup + use: "backup" + short: "Back up your beads database" + group: "sync" + hidden: false + runnable: false + long: + | Back up your beads database for off-machine recovery. + | + | This is a Dolt-native database backup. It preserves the database state, + | including tables, branches, commit history, and working-set data. This is + | different from 'bd export', which writes issue records to JSONL for migration + | and interoperability. + | + | Commands: + | bd backup init Set up a backup destination (filesystem or DoltHub) + | bd backup sync Push to configured backup destination + | bd backup restore [path] Restore from a backup directory + | bd backup remove Remove backup destination + | bd backup status Show backup status + | + | DoltHub is recommended for cloud backup: + | bd backup init https://doltremoteapi.dolthub.com// + | Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication. + | + | Auto-backup default: + | When backup.enabled is unset, auto-backup turns ON in embedded mode if a + | git remote exists, and stays OFF in sql-server / shared-server mode. In + | server mode many bd clients share one Dolt server, and each would register + | a server-side backup remote under the same name pointing at its own local + | dir and full-sync the whole database — a self-amplifying storm. To back up + | a shared server, run 'bd backup' explicitly (or set backup.enabled=true and + | coordinate destinations). 'bd config get backup.enabled' shows the effective + | value and its source. + | + | A proxied-server workspace is server mode too: auto-backup is OFF by + | default and backup.enabled=true opts in. Both the opt-in and the five + | backup verbs above are honored only on a managed-local proxied server — + | one bd started itself; on a Dolt server bd does not own they are refused. + +bd backup init + use: "init " + aliases: ["add"] + short: "Set up a Dolt backup destination" + hidden: false + runnable: true + long: + | Configure a filesystem path or URL as a backup destination. + | + | The path can be a local directory (external drive, NAS, Dropbox folder) or a + | DoltHub remote URL. If the destination was previously configured, it is + | updated to the new path. + | + | Filesystem examples: + | bd backup add /mnt/usb/beads-backup + | bd backup add ~/Dropbox/beads-backup + | + | DoltHub (recommended for cloud backup): + | bd backup add https://doltremoteapi.dolthub.com/myuser/beads-backup + | + | After adding, run 'bd backup sync' to push your data. + +bd backup remove + use: "remove" + aliases: ["rm"] + short: "Remove the configured backup destination" + hidden: false + runnable: true + long: + | Remove the configured backup destination. + | + | This unregisters the backup remote from Dolt and removes the local + | backup configuration. The backup data at the destination is not deleted. + +bd backup restore + use: "restore [path]" + short: "Restore database from a Dolt backup" + hidden: false + runnable: true + long: + | Restore the beads database from a Dolt-native backup. + | + | By default, reads from .beads/backup/ (or the configured backup directory). + | Optionally specify a path to a directory containing a Dolt backup. + | + | This restores a full database backup created by 'bd backup sync' or an + | equivalent Dolt backup. JSONL files produced by 'bd export' are issue exports, + | not restore targets for this command. + | + | Use --force to overwrite an existing database with the backup contents. + | + | The database must already be initialized (run 'bd init' first if needed). + | To initialize and restore in one step, use: bd init && bd backup restore + flag --force type=bool default="false" usage="Overwrite existing database with backup contents" + +bd backup status + use: "status" + short: "Show last backup status" + hidden: false + runnable: true + +bd backup sync + use: "sync" + short: "Push database to configured Dolt backup" + hidden: false + runnable: true + long: + | Sync the current beads database to the configured Dolt backup destination. + | + | This pushes the entire database state (all branches, full history) to the + | backup location configured with 'bd backup init'. + | + | The backup is atomic — if the sync fails, the previous backup state is preserved. + | + | Run 'bd backup init ' first to configure a destination. + +bd batch + use: "batch" + short: "Run multiple write operations in a single database transaction" + group: "maint" + hidden: false + runnable: true + long: + | Run multiple write operations in a single database transaction. + | + | Commands are read from stdin (one per line) or from a file via -f/--file. + | All operations execute inside a single dolt transaction: on any error the + | whole batch is rolled back, otherwise it is committed with one DOLT_COMMIT. + | + | This is intended for shell scripts that currently invoke 'bd' many times in + | a loop, which causes severe write amplification on a dolt sql-server backed + | by btrfs+compression. Batching collapses N invocations into one transaction + | and one dolt commit. + | + | Grammar (one command per line): + | close [reason...] + | update = [= ...] + | create + | dep add [type] + | dep remove + | #comment (blank lines and '# ...' comments are ignored) + | + | Supported 'update' keys: status, priority, title, assignee, force + | Supported dependency types: see 'bd dep add --help' (default: blocks) + | + | 'force' is not a field. An update whose status moves the issue into closed + | (or a configured done status) is refused when it still has open children or + | a live blocker, the same as 'bd close'; 'force=true' overrides that refusal. + | Because the batch is one transaction, an unforced refusal rolls back EVERY + | operation in the batch, not just the offending line. Note the asymmetry with + | 'close ', which does not apply that policy at all. + | + | Tokens are whitespace-separated. Double-quoted strings ("like this") may + | contain spaces; use \" to embed a quote and \\ for a backslash. + | + | Examples: + | # From a pipe + | bd list --status stale -q | awk '{print "close",$1," stale"}' | bd batch + | + | # From a file + | bd batch -f operations.txt + | + | # Inline + | printf 'close bd-1 done\nupdate bd-2 status=in_progress\n' | bd batch + | + | On success, exits 0 and prints a summary (or JSON with --json). On any error, + | rolls back the entire transaction and exits non-zero with the failing line. + | + | NOTE: This is a narrow subset. Commands like 'show', 'list', 'ready', 'sync', + | complex create flows, or any flag not listed above are NOT accepted. Use + | normal 'bd' subcommands for interactive/read operations. + flag --dry-run type=bool default="false" usage="Parse input and echo commands without executing" + flag --file -f type=string default="" usage="Read commands from file instead of stdin" + flag --message -m type=string default="" usage="DOLT_COMMIT message (default: 'bd: batch N ops by ')" + +bd blocked + use: "blocked" + short: "Show blocked issues" + hidden: false + runnable: true + flag --exclude-label type=stringSlice default="[]" usage="Exclude issues that have ANY of these labels" + flag --flags type=bool default="false" usage="Print the blocked flag for each id read from stdin (one per line) instead of the listing; with --json, [{\"id\",\"is_blocked\"}]" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL). Can combine with --label-any" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE). Can combine with --label" + flag --parent type=string default="" usage="Filter to descendants of this bead/epic" + flag --raw type=bool default="false" usage="With --flags: the stored blocked column itself, without the external-dependency policy (an unsatisfied external: blocker does not set it)" + +bd bootstrap + use: "bootstrap" + short: "Non-destructive database setup for fresh clones and recovery" + group: "setup" + hidden: false + runnable: true + long: + | Bootstrap sets up the beads database without destroying existing data. + | Unlike 'bd init --force', bootstrap will never delete existing issues. + | + | Bootstrap auto-detects the right action: + | • If sync.remote is configured: clones from the remote + | • If git origin has Dolt data (refs/dolt/data): clones from git and wires origin for future push/pull + | • If .beads/backup/*.jsonl exists: restores from backup + | • If .beads/issues.jsonl exists: imports from git-tracked JSONL + | • If no database exists: creates a fresh one — unless the workspace was already + | initialized (metadata.json has a project_id) and its server-mode database is + | missing, in which case bootstrap refuses rather than strand the existing data + | • If database already exists: validates and reports status + | + | If sync.remote points at a git repository, bootstrap verifies refs/dolt/data + | before cloning. Bootstrap exits non-zero when it cannot set up a database. + | + | This is the recommended command for: + | • Setting up beads on a fresh clone + | • Recovering after moving to a new machine + | • Repairing a broken database configuration + | + | Non-interactive mode (--non-interactive, --yes/-y, or BD_NON_INTERACTIVE=1): + | Skips the confirmation prompt before executing the bootstrap plan. + | Also auto-detected when stdin is not a terminal or CI=true is set. + | + | Examples: + | bd bootstrap # Auto-detect and set up + | bd bootstrap --dry-run # Show what would be done + | bd bootstrap --json # Output plan as JSON + | bd bootstrap --yes # Skip confirmation prompt + | + flag --dry-run type=bool default="false" usage="Show what would be done without doing it" + flag --non-interactive type=bool default="false" usage="Alias for --yes" + flag --yes -y type=bool default="false" usage="Skip confirmation prompts (for CI/automation)" + +bd branch + use: "branch [name]" + short: "List or create branches" + group: "sync" + hidden: false + runnable: true + long: + | List all branches or create a new branch. + | + | This command requires the Dolt storage backend. Without arguments, + | it lists all branches. With an argument, it creates a new branch. + | + | Examples: + | bd branch # List all branches + | bd branch feature-xyz # Create a new branch named feature-xyz + +bd children + use: "children " + short: "List child beads of a parent" + group: "issues" + hidden: false + runnable: true + long: + | List all beads that are children of the specified parent bead. + | + | This is a convenience alias for 'bd list --parent --status all'. + | Unlike plain 'bd list', children includes closed issues by default, + | since the primary use case is inspecting all work under a parent. + | + | Examples: + | bd children hq-abc123 # List all children of hq-abc123 + | bd children hq-abc123 --json # List children in JSON format + | bd children hq-abc123 --pretty # Show children in tree format + flag --pretty type=bool default="false" usage="Show children in tree format" + +bd close + use: "close [id...]" + aliases: ["done"] + short: "Close one or more issues" + group: "issues" + hidden: false + runnable: true + long: + | Close one or more issues. + | + | If no issue ID is provided, closes the last touched issue (from most recent + | create, update, show, or close operation). This fallback only applies in + | interactive sessions (stdin is a terminal); in scripts and agent sessions a + | missing ID is an error, so a command built from an empty variable cannot + | silently close an unrelated issue. Set BD_LAST_TOUCHED_FALLBACK=1 to allow + | the fallback anywhere, or =0 to disable it entirely. + | + | When closing multiple issues, provide one --reason for all IDs or repeat + | --reason once per ID. Reasons map positionally: the first --reason applies + | to the first ID, the second --reason to the second ID, regardless of where + | the flags appear in the command line. + | + | Closing a molecule's last open step also closes the molecule's root, with + | the reason "all steps complete", when the root is a poured molecule, a wisp, + | or an epic labeled "template"; any other epic stays open as close-eligible + | work. A root that would refuse its own close (an open blocker, a pin, another + | assignee) stays open with a warning, and the step's close stands. Under + | --json, the step's entry names the closed root (auto_closed_molecule) or the + | refusal (molecule_auto_close_refusal). --continue claims the next ready step + | in the molecule for you; add --no-auto to only show it. + flag --claim-next type=bool default="false" usage="Automatically claim the next highest priority available issue" + flag --comment type=string default="" hidden usage="Alias for --reason" + flag --continue type=bool default="false" usage="Auto-advance to next step in molecule" + flag --force -f type=bool default="false" usage="Force close pinned issues or unsatisfied gates" + flag --if-revision type=string default="" usage="Apply the write only if the issue's current revision equals this value (the decimal int64 from `bd show --json`'s \"revision\" field). One id only: refused before any write when combined with more than one issue id. A mismatch writes nothing and exits 13 (vs 1 for other failures)." + flag --message -m type=string default="" hidden usage="Alias for --reason (git commit convention)" + flag --no-auto type=bool default="false" usage="With --continue, show next step but don't claim it" + flag --reason -r type=string default="" usage="Reason for closing" + flag --reason-file type=string default="" usage="Read close reason from file (use - for stdin)" + flag --resolution type=string default="" hidden usage="Alias for --reason (Jira CLI convention)" + flag --session type=string default="" usage="Claude Code session ID (or set CLAUDE_SESSION_ID env var)" + flag --suggest-next type=bool default="false" usage="Show newly unblocked issues after closing" + +bd codex-hook + use: "codex-hook " + short: "Run an internal Codex lifecycle hook" + hidden: true + runnable: true + +bd comment + use: "comment [text...]" + short: "Add a comment to an issue" + group: "issues" + hidden: false + runnable: true + long: + | Add a comment to an issue. + | + | Shorthand for 'bd comments add "text"'. + | + | Examples: + | bd comment bd-123 "Working on this now" + | bd comment bd-123 Working on this now + | echo "comment from pipe" | bd comment bd-123 --stdin + | bd comment bd-123 --file notes.txt + | + | Note: "comment" (singular) only adds a comment — it has no "list" subcommand. + | To list comments on an issue, use the plural form: bd comments + flag --file type=string default="" annotations={cobra_annotation_mutually_exclusive=["stdin file"]} usage="Read comment text from file" + flag --stdin type=bool default="false" annotations={cobra_annotation_mutually_exclusive=["stdin file"]} usage="Read comment text from stdin" + +bd comments + use: "comments [issue-id]" + short: "View or manage comments on an issue" + group: "issues" + hidden: false + runnable: true + long: + | View or manage comments on an issue. + | + | Examples: + | # List all comments on an issue (issue id is required — there is no "comments list") + | bd comments bd-123 + | + | # List comments in JSON format + | bd comments bd-123 --json + | + | # Add a comment + | bd comments add bd-123 "This is a comment" + | + | # Add a comment from a file + | bd comments add bd-123 -f notes.txt + flag --local-time type=bool default="false" usage="Show timestamps in local time instead of UTC" + +bd comments add + use: "add [issue-id] [text...]" + short: "Add a comment to an issue" + hidden: false + runnable: true + long: + | Add a comment to an issue. + | + | Examples: + | # Add a comment + | bd comments add bd-123 "Working on this now" + | + | # Add a comment from a file + | bd comments add bd-123 -f notes.txt + flag --author -a type=string default="" usage="Add author to comment" + flag --file -f type=string default="" usage="Read comment text from file" + +bd comments list + use: "list" + short: "Invalid — use bd comments to list comments" + hidden: false + runnable: true + +bd compact + use: "compact" + short: "Squash old Dolt commits to reduce history size" + group: "maint" + hidden: false + runnable: true + long: + | Squash Dolt commits older than N days into a single commit. + | + | Recent commits (within the retention window) are preserved via cherry-pick. + | This reduces Dolt storage overhead from auto-commit history while keeping + | recent change tracking intact. + | + | For semantic issue compaction (summarizing closed issues), use 'bd admin compact'. + | For full history squash, use 'bd flatten'. + | + | How it works: + | 1. Identifies commits older than --days threshold + | 2. Creates a squashed base commit from all old history + | 3. Cherry-picks recent commits on top + | 4. Swaps main branch to the compacted version + | 5. Prunes remote-tracking refs (they would keep the old history alive; + | the next push or fetch re-creates them at the new tip) + | 6. Runs a full Dolt GC (all storage generations) to reclaim the old history + | + | The GC pass is a full collection: Dolt storage is generational, and a default + | GC never revisits data an earlier GC moved to the old generation. On any store + | that has been GC'd before (bd gc, or a previous flatten or compact), only a + | full collection reclaims the squashed history. A full GC can take minutes on + | multi-gigabyte stores. + | + | Examples: + | bd compact --dry-run # Preview: show commit breakdown + | bd compact --force # Squash commits older than 30 days + | bd compact --days 7 --force # Keep only last 7 days of history + | bd compact --days 90 --force # Conservative: squash 90+ day old commits + flag --days type=int default="30" usage="Keep commits newer than N days" + flag --dry-run type=bool default="false" usage="Preview without making changes" + flag --force -f type=bool default="false" usage="Confirm commit squash" + +bd completion + use: "completion" + short: "Generate the autocompletion script for the specified shell" + hidden: false + runnable: false + long: + | Generate the autocompletion script for bd for the specified shell. + | See each sub-command's help for details on how to use the generated script. + | + +bd completion bash + use: "bash" + short: "Generate the autocompletion script for bash" + hidden: false + runnable: true + long: + | Generate the autocompletion script for the bash shell. + | + | This script depends on the 'bash-completion' package. + | If it is not installed already, you can install it via your OS's package manager. + | + | To load completions in your current shell session: + | + | source <(bd completion bash) + | + | To load completions for every new session, execute once: + | + | #### Linux: + | + | bd completion bash > /etc/bash_completion.d/bd + | + | #### macOS: + | + | bd completion bash > $(brew --prefix)/etc/bash_completion.d/bd + | + | You will need to start a new shell for this setup to take effect. + | + flag --no-descriptions type=bool default="false" usage="disable completion descriptions" + +bd completion fish + use: "fish" + short: "Generate the autocompletion script for fish" + hidden: false + runnable: true + long: + | Generate the autocompletion script for the fish shell. + | + | To load completions in your current shell session: + | + | bd completion fish | source + | + | To load completions for every new session, execute once: + | + | bd completion fish > ~/.config/fish/completions/bd.fish + | + | You will need to start a new shell for this setup to take effect. + | + flag --no-descriptions type=bool default="false" usage="disable completion descriptions" + +bd completion powershell + use: "powershell" + short: "Generate the autocompletion script for powershell" + hidden: false + runnable: true + long: + | Generate the autocompletion script for powershell. + | + | To load completions in your current shell session: + | + | bd completion powershell | Out-String | Invoke-Expression + | + | To load completions for every new session, add the output of the above command + | to your powershell profile. + | + flag --no-descriptions type=bool default="false" usage="disable completion descriptions" + +bd completion zsh + use: "zsh" + short: "Generate the autocompletion script for zsh" + hidden: false + runnable: true + long: + | Generate the autocompletion script for the zsh shell. + | + | If shell completion is not already enabled in your environment you will need + | to enable it. You can execute the following once: + | + | echo "autoload -U compinit; compinit" >> ~/.zshrc + | + | To load completions in your current shell session: + | + | source <(bd completion zsh) + | + | To load completions for every new session, execute once: + | + | #### Linux: + | + | bd completion zsh > "${fpath[1]}/_bd" + | + | #### macOS: + | + | bd completion zsh > $(brew --prefix)/share/zsh/site-functions/_bd + | + | You will need to start a new shell for this setup to take effect. + | + flag --no-descriptions type=bool default="false" usage="disable completion descriptions" + +bd config + use: "config" + short: "Manage configuration settings" + group: "setup" + hidden: false + runnable: false + long: + | Manage configuration settings for external integrations and preferences. + | + | Configuration is stored per-project in the beads database and is version-control-friendly. + | + | Common namespaces: + | - export.* Auto-export settings (stored in config.yaml) + | - import.* JSONL import settings (stored in config.yaml) + | - jira.* Jira integration settings + | - linear.* Linear integration settings + | - github.* GitHub integration settings + | - gitlab.* GitLab integration settings + | - ado.* Azure DevOps integration settings + | - notion.* Notion integration settings + | - custom.* Custom integration settings + | - status.* Issue status configuration + | - claim.* Claim arbitration settings (pool-aware claiming) + | - doctor.suppress.* Suppress specific bd doctor warnings (GH#1095) + | - lint.* Additional per-type lint sections (stored in config.yaml) + | + | Auto-Export (config.yaml): + | Optional JSONL export to .beads/issues.jsonl after write commands (throttled). + | Useful for viewers (bv), interchange, and issue-level migration; not a backup. + | It is not cross-machine sync; use bd dolt push/pull with a Dolt remote. + | Disabled by default. Enable only for integrations that need fresh JSONL. + | Auto-staging is separate and disabled by default. + | + | Keys: + | export.auto Enable/disable auto-export (default: false) + | export.path Output filename relative to .beads/ (default: issues.jsonl) + | export.interval Minimum time between exports (default: 60s) + | export.git-add Auto-stage the export file (default: false) + | + | Auto-Import (config.yaml): + | Reads .beads/issues.jsonl by default when a JSONL import path is implied. + | Use a relative filename/path so the import stays within the project .beads/ + | directory and remains portable across machines. + | + | Keys: + | import.path Input filename relative to .beads/ (default: issues.jsonl) + | + | Custom Status States: + | You can define custom status states for multi-step pipelines using the + | status.custom config key. Statuses should be comma-separated. + | + | Example: + | bd config set status.custom "awaiting_review,awaiting_testing,awaiting_docs" + | + | This enables issues to use statuses like 'awaiting_review' in addition to + | the built-in statuses (open, in_progress, blocked, deferred, closed). + | + | Claim Pools: + | A dispatcher can pre-assign issues to a pool pseudo-assignee (e.g. + | "fable-crew") and let any actor take them with --claim. List the pool + | aliases in the claim.pools config key, comma-separated: + | + | bd config set claim.pools "fable-crew,night-crew" + | + | Issues assigned to a real actor (or to an alias not in the list) keep + | their anti-steal protection. Pool takes carry the normal lease; note + | that if a taker's lease expires, bd reclaim returns the issue to the + | unassigned pool, not to the pool alias it was dispatched to. + | + | Suppressing Doctor Warnings: + | Suppress specific bd doctor warnings by check name slug: + | bd config set doctor.suppress.pending-migrations true + | bd config set doctor.suppress.git-hooks true + | Check names are converted to slugs: "Git Hooks" → "git-hooks". + | Only warnings are suppressed (errors and passing checks always show). + | To unsuppress: bd config unset doctor.suppress. + | + | Examples: + | bd config set export.auto true # Enable auto-export for viewer integrations + | bd config set export.path "beads.jsonl" # Custom export filename + | bd config set import.path "beads.jsonl" # Custom import filename + | bd config set export.git-add true # Also stage the export file + | bd config set jira.url "https://company.atlassian.net" + | bd config set jira.project "PROJ" + | bd config set status.custom "awaiting_review,awaiting_testing" + | bd config set claim.pools "fable-crew,night-crew" # Pool aliases claimable by any actor + | bd config set doctor.suppress.pending-migrations true + | bd config set dolt.debug true # Enable Dolt sql-server debug mode (loglevel=debug, --prof cpu) + | bd config set dolt.local-only true # Skip wiring a Dolt sync remote during bd init + | bd config set lint.sections.epic "Standards scorecard, Cost" # Extra sections bd lint requires for epics + | bd config get export.auto + | bd config list + | bd config unset jira.url + +bd config apply + use: "apply" + short: "Reconcile system state to match configuration" + hidden: false + runnable: true + long: + | Reconcile actual system state to match declared configuration. + | + | Runs drift detection and then fixes any mismatches it finds: + | + | - hooks Reinstall git hooks if missing or outdated + | - remote Add/update Dolt origin remote to match federation.remote + | - server Start Dolt server if dolt.shared-server is enabled + | + | This command is idempotent — safe to run multiple times. Use --dry-run + | to preview what would change without making modifications. + | + | Examples: + | bd config apply + | bd config apply --dry-run + | bd config apply --json + flag --dry-run type=bool default="false" usage="Show what would change without making modifications" + +bd config drift + use: "drift" + short: "Detect config-vs-reality inconsistencies" + hidden: false + runnable: true + long: + | Detect drift between declared configuration and actual system state. + | + | This is a read-only diagnostic that answers "is my environment consistent + | with my config?" — no mutations are performed. + | + | Checks: + | - hooks Git hooks installed and up-to-date + | - remote Dolt remote matches federation.remote config + | - server Server state matches dolt.shared-server config + | + | Exit codes: + | 0 No drift detected (all checks ok/info/skipped) + | 1 Drift detected (at least one check has status "drift") + | + | Examples: + | bd config drift + | bd config drift --json + +bd config get + use: "get " + short: "Get a configuration value" + hidden: false + runnable: true + +bd config list + use: "list" + short: "List all configuration" + hidden: false + runnable: true + +bd config set + use: "set " + short: "Set a configuration value" + hidden: false + runnable: true + flag --force-git-tracked type=bool default="false" usage="Allow writing secret keys to git-tracked config files (use with caution)" + +bd config set-many + use: "set-many ..." + short: "Set multiple configuration values in one operation" + hidden: false + runnable: true + long: + | Set multiple configuration values at once with a single auto-commit and auto-push. + | + | Each argument must be in key=value format. All values are validated before + | any writes occur. This is faster and less noisy than separate 'bd config set' + | calls, especially in CI. + | + | Examples: + | bd config set-many ado.state_map.open=New ado.state_map.closed=Closed + | bd config set-many jira.url=https://example.atlassian.net jira.project=PROJ + flag --force-git-tracked type=bool default="false" usage="Allow writing secret keys to git-tracked config files (use with caution)" + +bd config show + use: "show" + short: "Show all effective configuration with provenance" + hidden: false + runnable: true + long: + | Display a unified view of all effective configuration across all sources + | with annotations showing where each value comes from. + | + | Sources (by precedence for Viper-managed keys): + | - env Environment variable (BD_* or BEADS_*) + | - config.yaml Project config file (.beads/config.yaml) + | - default Built-in default value + | + | Additional sources: + | - metadata Connection settings from .beads/metadata.json + | - database Integration config stored in the Dolt database + | - git Git config (e.g., beads.role) + | + | Examples: + | bd config show + | bd config show --json + | bd config show --source config.yaml + flag --source type=string default="" usage="Filter by source (e.g., config.yaml, env, default, metadata, database, git)" + +bd config unset + use: "unset " + short: "Delete a configuration value" + hidden: false + runnable: true + +bd config validate + use: "validate" + short: "Validate sync-related configuration" + hidden: false + runnable: true + long: + | Validate sync-related configuration settings. + | + | Checks: + | - federation.sovereignty is valid (T1, T2, T3, T4, or empty) + | - federation.remote is set for Dolt sync + | - Remote URL format is valid (dolthub://, gs://, s3://, az://, file://) + | - routing.mode is valid (auto, maintainer, contributor, explicit) + | + | Examples: + | bd config validate + | bd config validate --json + +bd conflicts + use: "conflicts" + short: "Inspect and resolve live merge conflicts" + group: "sync" + hidden: false + runnable: false + long: + | Inspect and resolve the merge conflicts sitting in the working set. + | + | Conflicts appear when a pull or merge brought in changes that collide with + | local ones and could not be settled automatically. These commands present them + | per issue and per field, and resolve them without the raw dolt CLI. + | + | Examples: + | bd conflicts list # which tables and issues are conflicted + | bd conflicts show # every conflicted row, field by field + | bd conflicts show bd-1234 # one issue + | bd conflicts resolve bd-1234 --ours # keep our side of one issue + | bd conflicts resolve --all --theirs # take their side of everything + +bd conflicts list + use: "list" + short: "List tables and issues with live merge conflicts" + hidden: false + runnable: true + +bd conflicts resolve + use: "resolve [...]" + short: "Resolve merge conflicts with --ours or --theirs" + hidden: false + runnable: true + long: + | Resolve live merge conflicts, then conclude the merge with a commit. + | + | Named issue IDs are resolved row by row, leaving every other conflicted row + | alone. --all resolves whole tables at once (dolt's own table-level + | resolution). The merge is committed only once NO conflicts remain, so a + | partial resolution leaves the merge open for the next pass. + | + | Row-by-row resolution requires the row to exist on both sides: when one side + | deleted it, resolve that table wholesale or edit the row directly. + | + | Examples: + | bd conflicts resolve bd-1234 --ours # keep our side of one issue + | bd conflicts resolve bd-1234 bd-5678 --theirs # take their side of two + | bd conflicts resolve --all --ours # every conflicted table + | bd conflicts resolve --all --table config --theirs + | bd conflicts resolve --conclude # commit an already-resolved merge + flag --all type=bool default="false" usage="Resolve whole tables instead of named issues" + flag --conclude type=bool default="false" usage="Commit a merge whose conflicts are already resolved" + flag --no-commit type=bool default="false" usage="Resolve without committing the merge" + flag --ours type=bool default="false" usage="Keep our side" + flag --strategy type=string default="" usage="Resolution strategy: ours|theirs" + flag --table type=string default="" usage="Table to resolve (default: issues)" + flag --theirs type=bool default="false" usage="Take their side" + +bd conflicts show + use: "show []" + short: "Show conflicted rows field by field (base/ours/theirs)" + hidden: false + runnable: true + long: + | Show each conflicted row with its fields side by side. + | + | Only fields where our side and their side disagree are shown; --all-fields + | shows every column. Without an issue ID, every conflicted row of every + | conflicted table is shown. + flag --all-fields type=bool default="false" usage="Show every column, not just the fields that diverged" + flag --table type=string default="" usage="Restrict to one conflicted table (default: all)" + +bd connect + use: "connect [url]" + short: "Attach this workspace to a remote bd serve over HTTP" + group: "setup" + hidden: false + runnable: true + annotations: {bd:skip_store="1"} + long: + | Attach this workspace to a remote bd serve over HTTP. + | + | bd connect verifies the server first (a handshake: api_version, wire + | revision, and — when --expect-project-id is given — workspace identity) and + | writes nothing until that succeeds. On success it writes the per-user + | activation sidecar (.beads/http_target.json, never git-tracked) and sets this + | workspace's metadata.json to backend "http". + | + | A credential, if the server requires one, is never read from a flag or + | recorded on disk by this command. It comes from the same ladder every http + | request does: the BEADS_HTTP_TOKEN environment variable, then + | BEADS_HTTP_TOKEN_COMMAND (a helper that prints a token), then the credentials + | file's [host:port] section, then no credential at all — which the tip OSS + | server's loopback-trust posture answers legitimately, not as a failure. Both + | variables name the server they are for, host[:port]=value (for example + | BEADS_HTTP_TOKEN=bd.example.com=); a bare value is refused, since it + | would be sent to whatever server the workspace's http_target.json names. + | + | --allow-plaintext IS recorded on disk, in the sidecar alongside the server + | url: once granted here, a credential may cross this one server in the clear + | on every later command too, not only this one, without re-passing the flag + | or setting BEADS_HTTP_ALLOW_INSECURE=1 yourself. A later connect to a + | DIFFERENT url starts that grant over; it is never carried forward to a + | server it was not given for. + | + | Examples: + | bd connect http://127.0.0.1:8080 # loopback, no TLS needed + | bd connect https://bd.example.com --expect-project-id p1 # pin workspace identity + | bd connect https://bd.example.com --ca-file ./ca.pem # trust only this CA + | bd connect --clear # forget the server, restoring the previous backend + | + flag --allow-plaintext type=bool default="false" usage="allow a plain http connection to a non-loopback host; remembered in the sidecar for every later command against this server" + flag --ca-file type=string default="" usage="trust ONLY this PEM file for this server (resolved and recorded as an absolute path)" + flag --clear type=bool default="false" usage="remove this workspace's http activation sidecar and restore its previous backend selection in metadata.json" + flag --expect-project-id type=string default="" usage="pin the workspace identity the server must present at every handshake" + flag --force type=bool default="false" usage="allow switching a workspace that already selects a different backend" + +bd context + use: "context" + short: "Show effective backend identity and repository context" + group: "setup" + hidden: false + runnable: true + long: + | Show the effective backend identity information including repository paths, + | backend configuration, and sync settings. + | + | This command reads directly from config files and does not require the + | database to be open, making it useful for diagnostics in degraded states. + | + | Examples: + | bd context # Show context information + | bd context --json # Output in JSON format + | + +bd cook + use: "cook " + short: "Compile a formula into a proto (ephemeral by default)" + hidden: false + runnable: true + long: + | Cook transforms a .formula.json file into a proto. + | + | By default, cook outputs the resolved formula as JSON to stdout for + | ephemeral use. The output can be inspected, piped, or saved to a file. + | + | Two cooking modes are available: + | + | COMPILE-TIME (default, --mode=compile): + | Produces a proto with {{variable}} placeholders intact. + | Use for: modeling, estimation, contractor handoff, planning. + | Variables are NOT substituted - the output shows the template structure. + | + | RUNTIME (--mode=runtime or when --var flags provided): + | Produces a fully-resolved proto with variables substituted. + | Use for: final validation before pour, seeing exact output. + | Requires all variables to have values (via --var or defaults). + | + | Formulas are high-level workflow templates that support: + | - Variable definitions with defaults and validation + | - Step definitions that become issue hierarchies + | - Composition rules for bonding formulas together + | - Inheritance via extends + | + | The --persist flag enables the legacy behavior of writing the proto + | to the database. This is useful when you want to reuse the same + | proto multiple times without re-cooking. + | + | For most workflows, prefer ephemeral protos: pour and wisp commands + | accept formula names directly and cook inline. + | + | Examples: + | bd cook mol-feature.formula.json # Compile-time: keep {{vars}} + | bd cook mol-feature --var name=auth # Runtime: substitute vars + | bd cook mol-feature --mode=runtime --var name=auth # Explicit runtime mode + | bd cook mol-feature --dry-run # Preview steps + | bd cook mol-release.formula.json --persist # Write to database + | bd cook mol-release.formula.json --persist --force # Replace existing + | + | Output (default): + | JSON representation of the resolved formula with all steps. + | + | Output (--persist): + | Creates a proto bead in the database with: + | - ID matching the formula name (e.g., mol-feature) + | - The "template" label for proto identification + | - Child issues for each step + | - Dependencies matching depends_on relationships + flag --dry-run type=bool default="false" usage="Preview what would be created" + flag --force type=bool default="false" usage="Replace existing proto if it exists (requires --persist)" + flag --mode type=string default="" usage="Cooking mode: compile (keep placeholders) or runtime (substitute vars)" + flag --persist type=bool default="false" usage="Persist proto to database (legacy behavior)" + flag --prefix type=string default="" usage="Prefix to prepend to proto ID (e.g., 'gt-' creates 'gt-mol-feature')" + flag --search-path type=stringSlice default="[]" usage="Additional paths to search for formula inheritance" + flag --var type=stringArray default="[]" usage="Variable substitution (key=value), enables runtime mode" + +bd count + use: "count" + short: "Count issues matching filters" + group: "views" + hidden: false + runnable: true + long: + | Count issues matching the specified filters. + | + | By default, returns the total count of issues matching the filters. + | Use --by-* flags to group counts by different attributes. + | + | Examples: + | bd count # Count all issues + | bd count --status open # Count open issues + | bd count --by-status # Group count by status + | bd count --by-priority # Group count by priority + | bd count --by-type # Group count by issue type + | bd count --by-assignee # Group count by assignee + | bd count --by-label # Group count by label + | bd count --assignee alice --by-status # Count alice's issues by status + | bd count --include-infra # Count issues + wisps tier (matches 'bd list --include-infra --all' cardinality) + | bd count --type task --include-ephemeral # Count tasks in the durable AND wisps tiers + | + flag --assignee -a type=string default="" usage="Filter by assignee" + flag --by-assignee type=bool default="false" usage="Group count by assignee" + flag --by-label type=bool default="false" usage="Group count by label" + flag --by-priority type=bool default="false" usage="Group count by priority" + flag --by-status type=bool default="false" usage="Group count by status" + flag --by-type type=bool default="false" usage="Group count by issue type" + flag --closed-after type=string default="" usage="Filter issues closed after date (YYYY-MM-DD or RFC3339)" + flag --closed-before type=string default="" usage="Filter issues closed before date (YYYY-MM-DD or RFC3339)" + flag --created-after type=string default="" usage="Filter issues created after date (YYYY-MM-DD or RFC3339)" + flag --created-before type=string default="" usage="Filter issues created before date (YYYY-MM-DD or RFC3339)" + flag --desc-contains type=string default="" usage="Filter by description substring" + flag --empty-description type=bool default="false" usage="Filter issues with empty description" + flag --exclude-status type=stringSlice default="[]" usage="Exclude statuses from the count (comma-separated or repeatable)" + flag --exclude-type type=stringSlice default="[]" usage="Exclude issue types from the count (comma-separated or repeatable)" + flag --has-metadata-key type=string default="" usage="Filter issues that have this metadata key set" + flag --id type=string default="" usage="Filter by specific issue IDs (comma-separated)" + flag --include-ephemeral type=bool default="false" usage="Include ephemeral wisp-plane rows in the count (normally hidden), without lifting type exclusions" + flag --include-infra type=bool default="false" usage="Include infrastructure beads and the wisps tier (matches 'bd list --include-infra --all' cardinality)" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL)" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE)" + flag --metadata-field type=stringArray default="[]" usage="Filter by metadata field (key=value, repeatable)" + flag --no-assignee type=bool default="false" usage="Filter issues with no assignee" + flag --no-labels type=bool default="false" usage="Filter issues with no labels" + flag --no-parent type=bool default="false" usage="Count only top-level issues (exclude child issues)" + flag --notes-contains type=string default="" usage="Filter by notes substring" + flag --parent type=string default="" usage="Filter by parent issue ID (count children of specified issue)" + flag --priority -p type=int default="0" usage="Filter by priority (0-4: 0=critical, 1=high, 2=medium, 3=low, 4=backlog)" + flag --priority-max type=int default="0" usage="Filter by maximum priority (inclusive)" + flag --priority-min type=int default="0" usage="Filter by minimum priority (inclusive)" + flag --status -s type=string default="" usage="Filter by stored status (open, in_progress, blocked, deferred, closed). Note: dependency-blocked issues use 'bd blocked'" + flag --title type=string default="" usage="Filter by title text (case-insensitive substring match)" + flag --title-contains type=string default="" usage="Filter by title substring" + flag --type -t type=string default="" usage="Filter by type (bug, feature, task, epic, chore, decision, merge-request, molecule, gate)" + flag --updated-after type=string default="" usage="Filter issues updated after date (YYYY-MM-DD or RFC3339)" + flag --updated-before type=string default="" usage="Filter issues updated before date (YYYY-MM-DD or RFC3339)" + +bd create + use: "create [title]" + aliases: ["new"] + short: "Create a new issue (or batch from markdown/graph JSON)" + group: "issues" + hidden: false + runnable: true + flag --acceptance type=string default="" usage="Acceptance criteria" + flag --allow-empty-description type=bool default="false" usage="Allow empty description input from stdin or file" + flag --append-notes type=string default="" usage="Append to existing notes (with newline separator)" + flag --assignee -a type=string default="" usage="Assignee" + flag --body type=string default="" hidden annotations={cobra_annotation_mutually_exclusive=["stdin body"]} usage="Alias for --description (GitHub CLI convention)" + flag --body-file type=string default="" annotations={cobra_annotation_mutually_exclusive=["stdin body-file"]} usage="Read description from file (use - for stdin)" + flag --context type=string default="" usage="Additional context for the issue" + flag --defer type=string default="" usage="Defer until date (issue hidden from bd ready until then). Same formats as --due" + flag --deps type=stringSlice default="[]" usage="Dependencies as 'type:id' or bare 'id'. Bare 'id', 'depends-on:id', and 'blocked-by:id' all make THIS issue depend on id; 'blocks:id' reverses direction (id depends on this issue). E.g. 'blocked-by:bd-20,discovered-from:bd-15'" + flag --description -d type=string default="" annotations={cobra_annotation_mutually_exclusive=["stdin description"]} usage="Issue description" + flag --description-file type=string default="" hidden annotations={cobra_annotation_mutually_exclusive=["stdin description-file"]} usage="Alias for --body-file" + flag --design type=string default="" annotations={cobra_annotation_mutually_exclusive=["design design-file"]} usage="Design notes" + flag --design-file type=string default="" annotations={cobra_annotation_mutually_exclusive=["design design-file"]} usage="Read design from file (use - for stdin)" + flag --dry-run type=bool default="false" usage="Preview what would be created without actually creating" + flag --due type=string default="" usage="Due date/time. Formats: +6h, +1d, +2w, tomorrow, next monday, 2025-01-15" + flag --ephemeral type=bool default="false" usage="Create as ephemeral (short-lived, subject to TTL compaction)" + flag --estimate -e type=int default="0" usage="Time estimate in minutes (e.g., 60 for 1 hour)" + flag --event-actor type=string default="" usage="Entity URI who caused this event (requires --type=event)" + flag --event-category type=string default="" usage="Event category (e.g., patrol.muted, agent.started) (requires --type=event)" + flag --event-payload type=string default="" usage="Event-specific JSON data (requires --type=event)" + flag --event-target type=string default="" usage="Entity URI or bead ID affected (requires --type=event)" + flag --external-ref type=string default="" usage="External reference (e.g., 'gh-9', 'jira-ABC', Linear URL)" + flag --file -f type=string default="" usage="Create one issue per ## heading (### fields attach to that issue); for title plus description-from-file, use --body-file" + flag --force type=bool default="false" usage="Force creation even if prefix doesn't match database prefix" + flag --graph type=string default="" usage="Create a graph of issues with dependencies from JSON plan file" + flag --id type=string default="" usage="Explicit issue ID (e.g., 'bd-42' for partitioning)" + flag --label type=stringSlice default="[]" hidden usage="Alias for --labels" + flag --labels -l type=stringSlice default="[]" usage="Labels (comma-separated)" + flag --message -m type=string default="" hidden annotations={cobra_annotation_mutually_exclusive=["stdin message"]} usage="Alias for --description (git commit convention)" + flag --metadata type=string default="" usage="Set custom metadata (JSON object, or @file.json to read from file)" + flag --mol-type type=string default="" usage="Molecule type: swarm (multi-agent), patrol (recurring ops), work (default)" + flag --no-history type=bool default="false" usage="Skip Dolt commit history without making GC-eligible (for permanent agent beads)" + flag --no-inherit-labels type=bool default="false" usage="Don't inherit labels from parent issue" + flag --notes type=string default="" usage="Notes" + flag --parent type=string default="" usage="Parent issue ID for hierarchical child (e.g., 'bd-a3f8e9')" + flag --priority -p type=string default="2" usage="Priority (0-4 or P0-P4, 0=highest)" + flag --repo type=string default="" usage="Target repository for issue (overrides auto-routing)" + flag --silent type=bool default="false" usage="Output only the issue ID (for scripting)" + flag --skills type=string default="" usage="Required skills for this issue" + flag --spec-id type=string default="" usage="Link to specification document" + flag --status -s type=string default="" usage="Initial status" + flag --stdin type=bool default="false" annotations={cobra_annotation_mutually_exclusive=["stdin body-file" "stdin description-file" "stdin description" "stdin body" "stdin message"]} usage="Read description from stdin (alias for --body-file -)" + flag --storage-class type=string default="" usage="Storage class: versioned, unversioned, or ephemeral (default: storage-class. config, else versioned)" + flag --title type=string default="" usage="Issue title (alternative to positional argument)" + flag --type -t type=string default="task" usage="Issue type (bug|feature|task|epic|chore|decision|spike|story|milestone); custom types require types.custom config; aliases: enhancement/feat→feature, dec/adr→decision" + flag --validate type=bool default="false" usage="Validate description contains required sections for issue type" + flag --waits-for type=string default="" usage="Spawner issue ID to wait for (creates waits-for dependency for fanout gate)" + flag --waits-for-gate type=string default="all-children" usage="Gate type: all-children (wait for all) or any-children (wait for first)" + flag --wisp-type type=string default="" usage="Wisp type for TTL-based compaction: heartbeat, ping, patrol, gc_report, recovery, error, escalation" + +bd create-form + use: "create-form" + short: "Create a new issue using an interactive form" + group: "issues" + hidden: false + runnable: true + long: + | Create a new issue using an interactive terminal form. + | + | This command provides a user-friendly form interface for creating issues, + | with fields for title, description, type, priority, labels, and more. + | + | Use --parent to create a sub-issue under an existing parent issue. + | The child will get an auto-generated hierarchical ID (e.g., parent-id.1). + | + | The form uses keyboard navigation: + | - Tab/Shift+Tab: Move between fields + | - Enter: Submit the form (on the last field or submit button) + | - Ctrl+C: Cancel and exit + | - Arrow keys: Navigate within select fields + flag --parent type=string default="" usage="Parent issue ID for creating a hierarchical child (e.g., 'bd-a3f8e9')" + +bd cursor-hook + use: "cursor-hook " + short: "Run an internal Cursor agent lifecycle hook" + hidden: true + runnable: true + +bd db-proxy-child + use: "db-proxy-child" + short: "Internal: run as the database proxy child process" + hidden: true + runnable: true + long: + | db-proxy-child runs the long-lived per-rootDir TCP proxy that fronts a + | DatabaseServer. It is spawned by the parent bd process via fork+exec and is + | not intended to be invoked directly by users. + flag --backend type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="backend kind: external | local-server | local-shared-server" + flag --config type=string default="" usage="path to backend server config (e.g. dolt sql-server YAML)" + flag --database type=string default="" usage="database to select when running shutdown maintenance (local-server backend)" + flag --dolt-bin type=string default="" usage="path to the dolt executable" + flag --external-host type=string default="" usage="external backend: hostname or IP of the dolt sql-server" + flag --external-keep-alive type=duration default="0s" usage="external backend: TCP keepalive period (default 30s)" + flag --external-port type=int default="0" usage="external backend: TCP port of the dolt sql-server" + flag --external-socket-path type=string default="" usage="external backend: absolute path to a unix domain socket (overrides host/port)" + flag --idle-timeout type=duration default="0s" usage="idle timeout before shutdown (0 or negative = never shut down)" + flag --logpath type=string default="" usage="path the backend server should write its stdout/stderr to" + flag --port type=int default="0" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="port to listen on" + flag --root type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="root directory holding proxy.lock, proxy.pid, proxy.log" + flag --stop-epoch type=string default="" usage="stop epoch the parent observed at spawn; the proxy aborts before publishing if it advances" + +bd defer + use: "defer [id...]" + short: "Defer one or more issues for later" + hidden: false + runnable: true + long: + | Defer issues to put them on ice for later. + | + | Deferred issues are deliberately set aside - not blocked by anything specific, + | just postponed for future consideration. Unlike blocked issues, there's no + | dependency keeping them from being worked. Unlike closed issues, they will + | be revisited. + | + | Deferred issues don't show in 'bd ready' but remain visible in 'bd list'. + | + | A defer WITH a date is a snooze: once --until passes, the next ready-front + | read returns the issue to open automatically (same shape as 'bd undefer'). + | A defer WITHOUT a date is the indefinite icebox: it stays deferred until + | someone runs 'bd undefer'. + | + | Examples: + | bd defer bd-abc # Icebox indefinitely (until bd undefer) + | bd defer bd-abc --until=tomorrow # Snooze: auto-wakes once the date passes + | bd defer bd-abc --reason="waiting on API access" + | bd defer bd-abc bd-def # Defer multiple issues + flag --reason type=string default="" usage="Record why this issue is being deferred (appended to notes)" + flag --until type=string default="" usage="Defer until specific time (e.g., +1h, tomorrow, next monday)" + +bd delete + use: "delete [issue-id...]" + short: "Delete one or more issues and clean up references" + group: "issues" + hidden: false + runnable: true + long: + | Delete one or more issues and clean up all references to them. + | This command will: + | 1. Remove all dependency links (any type, both directions) involving the issues + | 2. Update text references to "[deleted:ID]" in directly connected issues + | 3. Permanently delete the issues from the database + | + | This is a destructive operation that cannot be undone. Use with caution. + | + | BATCH DELETION: + | Delete multiple issues at once: + | bd delete bd-1 bd-2 bd-3 --force + | + | Delete from file (one ID per line): + | bd delete --from-file deletions.txt --force + | + | Preview before deleting: + | bd delete --from-file deletions.txt --dry-run + | + | DEPENDENCY HANDLING (the same on a local database and against a team server): + | Default: Fails if any issue has dependents not in deletion set + | bd delete bd-1 bd-2 + | + | Cascade: Recursively delete all dependents + | bd delete bd-1 --cascade --force + | + | Force: Delete and orphan dependents + | bd delete bd-1 --force + flag --cascade type=bool default="false" usage="Recursively delete all dependent issues" + flag --dry-run type=bool default="false" usage="Preview what would be deleted without making changes" + flag --force -f type=bool default="false" usage="Actually delete (without this flag, shows preview)" + flag --from-file type=string default="" usage="Read issue IDs from file (one per line)" + flag --if-revision type=string default="" usage="Apply the write only if the issue's current revision equals this value (the decimal int64 from `bd show --json`'s \"revision\" field). One id only: refused before any write when combined with more than one issue id. A mismatch writes nothing and exits 13 (vs 1 for other failures)." + +bd dep + use: "dep [issue-id]" + short: "Manage dependencies" + group: "deps" + hidden: false + runnable: true + long: + | Manage dependencies between issues. + | + | When called with an issue ID and --blocks flag, creates a blocking dependency: + | bd dep --blocks + | + | This is equivalent to: + | bd dep add + | + | Examples: + | bd dep bd-xyz --blocks bd-abc # bd-xyz blocks bd-abc + | bd dep add bd-abc bd-xyz # Same as above (bd-abc depends on bd-xyz) + flag --blocks -b type=string default="" usage="Issue ID that this issue blocks (shorthand for: bd dep add )" + flag --no-cycle-check type=bool default="false" usage="Skip the post-add cycle warning (the per-edge cycle check still runs)" + +bd dep add + use: "add [issue-id] [depends-on-id]" + short: "Add a dependency" + hidden: false + runnable: true + long: + | Add a dependency between two issues. + | + | The depends-on-id can be provided as: + | - A positional argument: bd dep add issue-123 issue-456 + | - A flag: bd dep add issue-123 --blocked-by issue-456 + | - A flag: bd dep add issue-123 --depends-on issue-456 + | + | The --blocked-by and --depends-on flags are aliases and both mean "issue-123 + | depends on (is blocked by) the specified issue." + | + | The depends-on-id can be: + | - A local issue ID (e.g., bd-xyz) + | - An external reference: external:: + | + | For bulk wiring, pass newline-delimited JSON with --file. Each line must be an + | object with "from" and "to" fields, and may include "type". The aliases + | "issue_id" and "depends_on_id" are also accepted. Use --file - to read stdin. + | + | External references are stored as-is and resolved at query time using + | the external_projects config. They block the issue until the capability + | is "shipped" in the target project. + | + | With no -t/--type the edge is created as type=blocks, which excludes the + | dependent from bd ready. When stderr is an interactive terminal, an advisory + | note says so once per command; it is silent for scripted and agent callers + | (non-TTY stderr) and can be turned off with --quiet or BD_NO_DEP_TYPE_WARNING=1. + | + | Examples: + | bd dep add bd-42 bd-41 # Positional args + | bd dep add bd-42 --blocked-by bd-41 # Flag syntax (same effect) + | bd dep add bd-42 --depends-on bd-41 # Alias (same effect) + | bd dep add gt-xyz external:beads:mol-run-assignee # Cross-project dependency + | bd dep add bd-42 bd-41 --no-cycle-check # Skip the post-add cycle warning + | bd dep add --file deps.jsonl # Bulk JSONL: {"from":"bd-42","to":"bd-41"} + flag --blocked-by type=string default="" usage="Issue ID that blocks the first issue (alternative to positional arg)" + flag --depends-on type=string default="" usage="Issue ID that the first issue depends on (alias for --blocked-by)" + flag --file type=string default="" usage="Read dependency edges from JSONL file, or '-' for stdin" + flag --no-cycle-check type=bool default="false" usage="On bulk --file adds, skip per-edge cycle checks and the post-add cycle warning (one final whole-graph check still runs before commit); on a single-edge add, skip only the post-add cycle warning" + flag --type -t type=string default="blocks" usage="Dependency type (blocks|tracks|related|parent-child|discovered-from|until|caused-by|validates|relates-to|supersedes); 'blocked-by' and 'depends-on' are accepted as aliases for 'blocks'" + +bd dep cycles + use: "cycles" + short: "Detect dependency cycles" + hidden: false + runnable: true + flag --include-tracks type=bool default="false" usage="Also walk 'tracks' edges: reports every cycle the default walk finds plus cycles that close through a tracks edge, never a loop made only of tracks edges (diagnostic for molecule-root deadlocks hidden by tracks-only propagation)" + flag --limit type=int default="50" usage="Maximum cycles to print (0 = no limit); the reported total is always the whole count, and --json is never truncated" + +bd dep list + use: "list [issue-id...]" + short: "List dependencies or dependents of one or more issues" + hidden: false + runnable: true + long: + | List dependencies or dependents of one or more issues with optional type filtering. + | + | By default shows dependencies (what issues depend on). Use --direction to control: + | - down: Show dependencies (what this issue depends on) - default + | - up: Show dependents (what depends on this issue) + | + | Multiple IDs can be provided for batch dep listing. With --json, the output + | is a flat array of dependency records across all requested issues. + | + | Use --type to filter by dependency type (e.g., tracks, blocks, parent-child). + | + | Examples: + | bd dep list gt-abc # Show what gt-abc depends on + | bd dep list gt-abc gt-def # Batch: deps for both issues + | bd dep list gt-abc --direction=up # Show what depends on gt-abc + | bd dep list gt-abc --direction=up -t tracks # Show what tracks gt-abc (convoy tracking) + flag --direction type=string default="down" usage="Direction: 'down' (dependencies), 'up' (dependents)" + flag --type -t type=string default="" usage="Filter by dependency type (e.g., tracks, blocks, parent-child)" + +bd dep relate + use: "relate " + short: "Create a bidirectional relates_to link between issues" + hidden: false + runnable: true + long: + | Create a loose 'see also' relationship between two issues. + | + | The relates_to link is bidirectional - both issues will reference each other. + | This enables knowledge graph connections without blocking or hierarchy. + | + | Examples: + | bd relate bd-abc bd-xyz # Link two related issues + | bd relate bd-123 bd-456 # Create see-also connection + +bd dep remove + use: "remove [issue-id] [depends-on-id]" + aliases: ["rm"] + short: "Remove a dependency" + hidden: false + runnable: true + +bd dep tree + use: "tree [issue-id]" + short: "Show dependency tree" + hidden: false + runnable: true + long: + | Show dependency tree rooted at the given issue. + | + | By default, shows dependencies (what blocks this issue). Use --direction to control: + | - down: Show dependencies (what blocks this issue) - default + | - up: Show dependents (what this issue blocks) + | - both: Show full graph in both directions + | + | Examples: + | bd dep tree gt-0iqq # Show what blocks gt-0iqq + | bd dep tree gt-0iqq --direction=up # Show what gt-0iqq blocks + | bd dep tree gt-0iqq --status=open # Only show open issues + | bd dep tree gt-0iqq --depth=3 # Limit to 3 levels deep + | + | A node reached by two paths is shown ONCE, under the first path that got + | there, and a cycle simply ends the descent. --show-all-paths is a deprecated + | no-op; use 'bd dep cycles' to find circular dependencies. + | + | --max-rows / BEADS_MAX_ROWS caveat: the tree walk has no query filter to + | thread the cap through, so the full tree is always built first and the + | node count is checked afterward (post-hoc), not during the walk. The cap is + | honored on the --proxied-server route too, which it was not before. + flag --direction type=string default="" usage="Tree direction: 'down' (dependencies), 'up' (dependents), or 'both'" + flag --format type=string default="" usage="Output format: 'mermaid' for Mermaid.js flowchart" + flag --max-depth -d type=int default="50" usage="Maximum tree depth to display (safety limit)" + flag --max-rows type=int default="0" usage="Hard upper bound on rows returned. Returns a non-zero exit (code 2) and an error to stderr if exceeded. 0 disables (the default). Overrides BEADS_MAX_ROWS for this invocation. Useful in CI/agent rigs that want a circuit breaker against pathological queries. Honored on both the direct and the --proxied-server route." + flag --reverse type=bool default="false" usage="Show dependent tree (deprecated: use --direction=up)" + flag --show-all-paths type=bool default="false" usage="Deprecated no-op: accepted and ignored. A node reached by two paths is shown once, under the first." + flag --status type=string default="" usage="Filter to only show issues with this status (open, in_progress, blocked, deferred, closed)" + +bd dep unrelate + use: "unrelate " + short: "Remove a relates_to link between issues" + hidden: false + runnable: true + long: + | Remove a relates_to relationship between two issues. + | + | Removes the link in both directions. + | + | Example: + | bd unrelate bd-abc bd-xyz + +bd diff + use: "diff " + short: "Show changes between two commits or branches" + group: "views" + hidden: false + runnable: true + long: + | Show the differences in issues between two commits or branches. + | + | The refs can be: + | - Commit hashes (e.g., abc123def) + | - Branch names (e.g., main, feature-branch) + | - Special refs like HEAD, HEAD~1 + | + | Examples: + | bd diff main feature-branch # Compare main to feature branch + | bd diff HEAD~5 HEAD # Show changes in last 5 commits + | bd diff abc123 def456 # Compare two specific commits + +bd doctor + use: "doctor [path]" + short: "Check and fix beads installation health (start here)" + group: "maint" + hidden: false + runnable: true + annotations: {bd:skip_store="1"} + long: + | Sanity check the beads installation for the current directory or specified path. + | + | This command checks: + | - If .beads/ directory exists + | - Database version and migration status + | - Schema compatibility (all required tables and columns present) + | - Whether using hash-based vs sequential IDs + | - If CLI version is current (checks GitHub releases) + | - If Claude plugin is current (when running in Claude Code) + | - File permissions + | - Circular dependencies + | - Git hooks (pre-commit, post-merge, pre-push) + | - .beads/.gitignore up to date + | - Metadata.json version tracking (LastBdVersion field) + | + | Storage Availability: + | Full diagnostics, --perf, --deep, --server, --migration, and + | --check=validate currently require Dolt server mode. Embedded Dolt + | supports --check=artifacts, --check=conventions, and + | --check=pollution. --check-health has a limited hook-health fallback. + | Unsupported combinations return a notice without changing storage. + | + | Performance Mode (--perf): + | Run performance diagnostics on your database: + | - Times key operations (bd ready, bd list, bd show, etc.) + | - Collects system info (OS, arch, database stats) + | - Generates CPU profile for analysis + | - Outputs shareable report for bug reports + | + | Export Mode (--output): + | Save diagnostics to a JSON file for historical analysis and bug reporting. + | Includes timestamp and platform info for tracking intermittent issues. + | + | Specific Check Mode (--check): + | Run a specific check in detail. Available checks: + | - artifacts: Detect and optionally clean beads classic artifacts + | (stale JSONL, SQLite files, cruft .beads dirs). Use with --clean. + | - conventions: Check for convention drift (lint warnings, stale + | issues, orphaned issues). Advisory only - warns, never blocks. + | - pollution: Detect and optionally clean test issues from database + | - validate: Run focused data-integrity checks (duplicates, orphaned + | deps, test pollution, git conflicts). Use with --fix to auto-repair. + | + | Deep Validation Mode (--deep): + | Validate full graph integrity. May be slow on large databases. + | Additional checks: + | - Parent consistency: All parent-child deps point to existing issues + | - Dependency integrity: All deps reference valid issues + | - Epic completeness: Find epics ready to close (all children closed as + | completed work; duplicate/wontfix/superseded closes do not count) + | - Agent bead integrity: Agent beads have valid state values + | - Mail thread integrity: Thread IDs reference existing issues + | - Molecule integrity: Molecules have valid parent-child structures + | + | Server Mode (--server): + | Run health checks for Dolt server mode connections (bd-dolt.2.3): + | - Server reachable: Can connect to configured host:port? + | - Dolt version: Is it a Dolt server (not vanilla MySQL)? + | - Database exists: Does the 'beads' database exist? + | - Schema compatible: Can query beads tables? + | - Connection pool: Pool health metrics + | + | Legacy Dolt Migration Validation Mode (--migration): + | Retained for older SQLite-to-Dolt migration workflows and available only in + | Dolt server mode. It is not the migration path for removed backends + | (PostgreSQL, MySQL, SQLite); those fail closed with export/import guidance. + | Combine + | with --json for machine-parseable diagnostic output. + | + | Agent Mode (--agent): + | Output diagnostics designed for AI agent consumption. Instead of terse + | pass/fail messages, each issue includes: + | - Observed state: what the system actually looks like + | - Expected state: what it should look like + | - Explanation: full prose context about the issue and why it matters + | - Commands: exact remediation commands to run + | - Source files: where in the codebase to investigate further + | - Severity: blocking (prevents operation), degraded (partial function), + | or advisory (informational only) + | ZFC-compliant: Go observes and reports, the agent decides and acts. + | Combine with --json for structured agent-facing output. + | + | Suppressing Warnings: + | Suppress specific warnings by setting doctor.suppress. config: + | bd config set doctor.suppress.pending-migrations true + | bd config set doctor.suppress.git-hooks true + | Check names are converted to slugs: "Git Hooks" → "git-hooks". + | Only warnings are suppressed; errors and passing checks always show. + | To unsuppress: bd config unset doctor.suppress. + | + | Examples: + | bd doctor # Check current directory + | bd doctor /path/to/repo # Check specific repository + | bd doctor --json # Machine-readable output + | bd doctor --agent # Agent-facing diagnostic output + | bd doctor --agent --json # Structured agent diagnostics (JSON) + | bd doctor --fix # Automatically fix issues (with confirmation) + | bd doctor --fix --yes # Automatically fix issues (no confirmation) + | bd doctor --fix -i # Confirm each fix individually + | bd doctor --fix --fix-child-parent # Also fix child→parent deps (opt-in) + | bd doctor --fix --force # Force repair even when database can't be opened + | bd doctor --fix --source=jsonl # Rebuild database from a JSONL export + | bd doctor --dry-run # Preview what --fix would do without making changes + | bd doctor --perf # Performance diagnostics + | bd doctor --output diagnostics.json # Export diagnostics to file + | bd doctor --check=artifacts # Show classic artifacts (JSONL, SQLite, cruft dirs) + | bd doctor --check=artifacts --clean # Delete safe-to-delete artifacts (with confirmation) + | bd doctor --check=conventions # Convention drift check (lint, stale, orphans) + | bd doctor --check=pollution # Show potential test issues + | bd doctor --check=pollution --clean # Delete test issues (with confirmation) + | bd doctor --check=validate # Data-integrity checks only + | bd doctor --check=validate --fix # Auto-fix data-integrity issues + | bd doctor --deep # Full graph integrity validation + | bd doctor --server # Dolt server mode health checks + | bd doctor --migration=pre # Legacy Dolt-server migration diagnostic + | bd doctor --migration=post # Legacy Dolt-server completion diagnostic + | bd doctor --migration=pre --json # Machine-parseable legacy diagnostic + flag --agent type=bool default="false" usage="Agent-facing diagnostic mode: rich context for AI agents (ZFC-compliant)" + flag --check type=string default="" usage="Run specific check in detail (e.g., 'pollution')" + flag --check-health type=bool default="false" usage="Quick health check for git hooks (silent on success)" + flag --clean type=bool default="false" usage="For pollution check: delete detected test issues" + flag --deep type=bool default="false" usage="Validate full graph integrity" + flag --dry-run type=bool default="false" usage="Preview fixes without making changes" + flag --fix type=bool default="false" usage="Automatically fix issues where possible" + flag --fix-child-parent type=bool default="false" usage="Remove child→parent dependencies (opt-in)" + flag --interactive -i type=bool default="false" usage="Confirm each fix individually" + flag --migration type=string default="" usage="Run legacy Dolt-server migration diagnostics: 'pre' or 'post'" + flag --orchestrator type=bool default="false" usage="Running in orchestrator multi-workspace mode (routes.jsonl is expected, higher duplicate tolerance)" + flag --orchestrator-duplicates-threshold type=int default="1000" usage="Duplicate tolerance threshold for orchestrator mode (wisps are ephemeral)" + flag --output -o type=string default="" usage="Export diagnostics to JSON file" + flag --perf type=bool default="false" usage="Run performance diagnostics and generate CPU profile" + flag --server type=bool default="false" usage="Run Dolt server mode health checks (connectivity, version, schema)" + flag --verbose -v type=bool default="false" usage="Show all checks (default shows only warnings/errors)" + flag --yes -y type=bool default="false" usage="Skip confirmation prompt (for non-interactive use)" + +bd dolt + use: "dolt" + short: "Configure Dolt database settings" + group: "setup" + hidden: false + runnable: false + long: + | Configure and manage Dolt database settings and server lifecycle. + | + | Beads runs Dolt embedded (in-process) by default: there is no sql-server and + | nothing is auto-started. A database only uses a dolt sql-server when it is + | configured for one: shared-server mode, an explicit server mode, or a + | non-localhost dolt_server_host. The server-only commands below fail with + | "not supported in embedded mode (no Dolt server)" on an embedded database. + | + | Server lifecycle (server mode only): + | bd dolt start Start the Dolt server for this project + | bd dolt stop Stop the Dolt server for this project + | + | Diagnostics (both modes): + | bd dolt status Show Dolt engine status (embedded: in-process, data dir) + | bd dolt show Show current Dolt configuration with connection test + | + | Configuration (server mode only): + | bd dolt set Set a configuration value + | bd dolt test Test server connection + | + | Version control: + | bd dolt commit Commit pending changes + | bd dolt push Push commits to Dolt remote + | bd dolt pull Pull commits from Dolt remote + | + | Remote management: + | bd dolt remote add Add a Dolt remote + | bd dolt remote list List configured remotes + | bd dolt remote remove Remove a Dolt remote + | + | Configuration keys for 'bd dolt set': + | database Database name (default: issue prefix or "beads") + | host Server host (default: 127.0.0.1) + | port Server port (auto-detected; override with bd dolt set port ) + | user MySQL user (default: root) + | data-dir Custom dolt data directory (absolute path; default: .beads/dolt) + | + | Remote server authentication (password + TLS) is NOT stored via 'bd dolt set' + | (keeps secrets out of metadata.json). Configure them with: + | + | BEADS_DOLT_PASSWORD Server password (highest priority) + | BEADS_DOLT_SERVER_TLS Enable TLS (set to "1" or "true") + | BEADS_DOLT_SERVER_USER MySQL user override (else use 'bd dolt set user') + | BEADS_CREDENTIALS_FILE Optional path to credentials file + | + | Default credentials file: ~/.config/beads/credentials (Linux/macOS) + | %APPDATA%\beads\credentials (Windows) + | Format (INI, section = host:port of the resolved connection): + | [127.0.0.1:3307] + | password = secret + | + | Password resolution: BEADS_DOLT_PASSWORD → credentials [host:port] → empty. + | Full reference: docs/architecture/dolt.md (Environment Variables / Credentials). + | + | Flags for 'bd dolt set': + | --update-config Also write to config.yaml for team-wide defaults + | + | Examples: + | bd dolt set database myproject + | bd dolt set host 192.168.1.100 --update-config + | bd dolt set data-dir /home/user/.beads-dolt/myproject + | export BEADS_DOLT_PASSWORD=... BEADS_DOLT_SERVER_TLS=1 + | bd dolt test + +bd dolt clean-databases + use: "clean-databases" + short: "Drop stale test databases from the Dolt server" + hidden: false + runnable: true + long: + | Identify and drop leftover test and agent databases that accumulate + | on the shared Dolt server from interrupted test runs and terminated agents. + | + | Stale database prefixes: testdb_*, beads_test*, beads_pt*, beads_vr*, doctest_*, doctortest_*, benchdb_* + | + | These waste server memory and can degrade performance under concurrent load. + | Use --dry-run to see what would be dropped without actually dropping. + | + | DROP DATABASE only marks a database as dropped; Dolt keeps its directory + | under .dolt_dropped_databases/ so it can be restored with + | CALL DOLT_UNDROP(name) until an explicit purge — disk is not reclaimed + | until then. Pass --purge-dropped to run CALL DOLT_PURGE_DROPPED_DATABASES() + | after cleanup. + | + | --purge-dropped is SERVER-GLOBAL and IRREVERSIBLE. Dolt has no way to scope + | the purge to only the databases this run dropped: it permanently deletes + | every dropped-but-not-yet-purged database on the server, including ones + | dropped by something else entirely (e.g. an operator's accidental + | DROP DATABASE on an unrelated database that was still recoverable via + | DOLT_UNDROP). It also purges pre-existing residue from earlier + | clean-databases runs even if this run finds no stale databases to drop. + | Only pass it when nothing else on the server may be relying on DOLT_UNDROP + | recovery. + flag --dry-run type=bool default="false" usage="Show what would be dropped without dropping" + flag --purge-dropped type=bool default="false" usage="After dropping, also run CALL DOLT_PURGE_DROPPED_DATABASES() — server-global and irreversible, see --help" + +bd dolt commit + use: "commit" + short: "Create a Dolt commit from pending changes" + hidden: false + runnable: true + long: + | Create a Dolt commit from any uncommitted changes in the working set. + | + | This is the primary commit point for batch mode. When auto-commit is set to + | "batch", changes accumulate in the working set across multiple bd commands and + | are committed together here with a descriptive summary message. + | + | Also useful before push operations that require a clean working set, or when + | auto-commit was off or changes were made externally. + | + | For more options (--stdin, custom messages), see: bd vc commit + flag --message -m type=string default="" usage="Commit message (default: auto-generated)" + +bd dolt killall + use: "killall" + short: "Kill all orphan Dolt server processes" + hidden: false + runnable: true + long: + | Find and kill orphan dolt sql-server processes not tracked by the + | canonical PID file for the current repo's Dolt data directory. + | + | Under an orchestrator, the canonical server lives at $GT_ROOT/.beads/. Any other + | dolt sql-server processes using that shared data directory are considered + | orphans and will be killed. + | + | In standalone mode, only dolt sql-server processes using the current + | project's Dolt data directory are eligible for cleanup. Other projects' + | servers are preserved. + +bd dolt pull + use: "pull" + short: "Pull commits from Dolt remote" + hidden: false + runnable: true + long: + | Pull commits from the configured Dolt remote into the local database. + | + | Requires a Dolt remote to be configured in the database directory. + | For Hosted Dolt, set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD environment + | variables for authentication. + | + | Use --remote to pull from a specific named remote instead of the default. + | The remote must already exist (see 'bd dolt remote add'). + | + | Use --strategy ours|theirs to resolve conflicts the auto-resolver declines + | (e.g. both sides edited the same issue since the last sync) instead of + | aborting the pull for manual resolution. Embedded storage only (#4992); on + | server-mode/sql-server storage use 'bd conflicts resolve' after a pull that + | reports conflicts. + flag --remote type=string default="" usage="Pull from a specific named remote instead of the default" + flag --strategy type=string default="" usage="Conflict resolution strategy for conflicts the auto-resolver declines: 'ours' or 'theirs' (embedded storage only, #4992)" + +bd dolt push + use: "push" + short: "Push commits to Dolt remote" + hidden: false + runnable: true + long: + | Push local Dolt commits to the configured remote. + | + | Requires a Dolt remote to be configured in the database directory. With no + | remote configured, bd can adopt one derived from git origin — only with + | consent: interactively, or via --yes; --no-adopt or BD_NO_REMOTE_ADOPT=1 + | disables adoption entirely. + | For Hosted Dolt, set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD environment + | variables for authentication. + | + | Use --force to overwrite remote changes (e.g., when the remote has + | uncommitted changes in its working set). + | + | Use --remote to push to a specific named remote instead of the default. + | The remote must already exist (see 'bd dolt remote add'). + flag --force type=bool default="false" usage="Force push (overwrite remote changes)" + flag --no-adopt type=bool default="false" usage="Never derive a Dolt remote from git origin (also BD_NO_REMOTE_ADOPT=1)" + flag --remote type=string default="" usage="Push to a specific named remote instead of the default" + flag --yes -y type=bool default="false" usage="Consent to adopting a Dolt remote derived from git origin when none is configured" + +bd dolt remote + use: "remote" + short: "Manage Dolt remotes" + hidden: false + runnable: false + long: + | Manage Dolt remotes for push/pull replication. + | + | Subcommands: + | add Add a new remote + | list List all configured remotes + | remove Remove a remote + | reset-data Replace a remote's data plane after a history squash + +bd dolt remote add + use: "add " + short: "Add a Dolt remote" + hidden: false + runnable: true + flag --allow-git-origin type=bool default="false" usage="Allow adding a Dolt remote whose URL matches the git origin (proceed with a warning instead of aborting)" + +bd dolt remote list + use: "list" + short: "List configured Dolt remotes" + hidden: false + runnable: true + +bd dolt remote remove + use: "remove " + short: "Remove a Dolt remote" + hidden: false + runnable: true + +bd dolt remote reset-data + use: "reset-data " + short: "Replace a remote's data plane in place after a history squash" + hidden: false + runnable: true + long: + | Replace a Dolt remote's stored data with a fresh copy of local HEAD. + | + | After a history squash (see the History Bloat recovery runbook), a plain + | 'bd dolt push --force' re-points the remote's refs but deletes nothing: + | Dolt remotes accumulate chunks monotonically, so the remote keeps the full + | pre-squash store. This command rebuilds the remote's data plane so it holds + | only live chunks: + | + | - Git-backed remotes (issue data riding a git remote under refs/dolt/data): + | deletes the Dolt data refs on the git remote, then force-pushes to + | rebuild a fresh store. Code branches are untouched. + | - Native file remotes (file:// paths): clears the store directory, then + | force-pushes to rebuild it. + | - Cloud/hosted remotes (aws://, gs://, dolthub://, ...): bd cannot clear + | the stored data safely — replace the remote with a fresh URL or prefix: + | bd dolt remote remove + | bd dolt remote add + | bd dolt push --force + | + | This rewrites the remote's data plane. Every other clone must re-clone from + | the reset remote (that is already true after the squash itself). Refuses to + | run with uncommitted working-set changes: the rebuilt remote holds exactly + | HEAD, and anything uncommitted would not be part of it. + | + | Examples: + | bd dolt remote reset-data origin # prompts for confirmation + | bd dolt remote reset-data origin --yes # no prompt (scripts, agents) + flag --yes -y type=bool default="false" usage="Skip the confirmation prompt (required in non-interactive use)" + +bd dolt set + use: "set " + short: "Set a Dolt configuration value" + hidden: false + runnable: true + long: + | Set a Dolt configuration value in metadata.json. + | + | Keys: + | database Database name (default: issue prefix or "beads") + | host Server host (default: 127.0.0.1) + | port Server port (auto-detected; override with bd dolt set port ) + | user MySQL user (default: root) + | data-dir Custom dolt data directory (absolute path; default: .beads/dolt) + | + | There is no 'password' or 'tls' key here on purpose — secrets and TLS must + | not land in metadata.json. Use environment variables or the credentials file: + | + | BEADS_DOLT_PASSWORD Server password (highest priority) + | BEADS_DOLT_SERVER_TLS Enable TLS ("1" or "true") + | BEADS_CREDENTIALS_FILE Optional override path for credentials + | + | Default credentials file: ~/.config/beads/credentials + | Format: + | [host:port] + | password = secret + | + | See: bd dolt --help and docs/architecture/dolt.md + | + | Use --update-config to also write to config.yaml for team-wide defaults. + | + | Examples: + | bd dolt set database myproject + | bd dolt set host 192.168.1.100 + | bd dolt set port 3307 --update-config + | bd dolt set data-dir /home/user/.beads-dolt/myproject + | export BEADS_DOLT_PASSWORD=... BEADS_DOLT_SERVER_TLS=1 + flag --update-config type=bool default="false" usage="Also write to config.yaml for team-wide defaults" + +bd dolt show + use: "show" + short: "Show current Dolt configuration with connection status" + hidden: false + runnable: true + +bd dolt start + use: "start" + short: "Start the Dolt SQL server for this project" + hidden: false + runnable: true + long: + | Start a dolt sql-server for the current beads project. + | + | The server runs in the background on a per-project port derived from the + | project path. PID and logs are stored in .beads/. + | + | The server auto-starts transparently when needed, so manual start is rarely + | required. Use this command for explicit control or diagnostics. + | + | Not available in proxied-server mode: there the proxy owns its dolt backend's + | lifecycle and starts it on demand, so a server started here would be a second, + | unsupervised one over the same data directory. Use 'bd dolt status' to see what + | is running and 'bd dolt stop' to shut it down. + +bd dolt status + use: "status" + short: "Show Dolt engine status" + hidden: false + runnable: true + long: + | Show the status of the Dolt engine for the current project. + | + | In embedded mode, reports that the Dolt engine runs in-process and shows + | the on-disk data directory. For beads-managed (local) servers, displays + | PID, port, and data directory from the local PID file. For externally- + | managed servers — a shared server (dolt.shared-server: true), a remote + | dolt_server_host, or a local server managed outside bd (dolt.auto-start: + | false, e.g. an orchestrator-shared sql-server) — pings the configured + | endpoint via SQL and reports reachability, server version, and database. + | + | In proxied-server mode, reports the proxy (the endpoint every bd command + | connects through) and the dolt backend behind it separately, read from the + | proxy's own process records. Reporting starts nothing: a quiesced workspace + | is shown as not running, and the next bd command launches it. + +bd dolt stop + use: "stop" + short: "Stop the Dolt SQL server for this project" + hidden: false + runnable: true + long: + | Stop the dolt sql-server managed by beads for the current project. + | + | This sends a graceful shutdown signal. The server will restart automatically + | on the next bd command unless auto-start is disabled. + | + | For a managed proxied server, --force can recover unverifiable or legacy + | process records (both the proxy and its backend) only after each live process + | executable is matched to bd or dolt and its command line ties it to this + | workspace. In that recovery path, force still refuses to signal a process + | whose executable identity cannot be matched to bd or dolt, or whose workspace + | scope cannot be established. + flag --force type=bool default="false" usage="Force stop (proxied recovery still requires a bd/dolt executable match)" + +bd dolt test + use: "test" + short: "Test connection to Dolt server" + hidden: false + runnable: true + long: + | Test the connection to the configured Dolt server. + | + | This verifies that: + | 1. The server is reachable at the configured host:port + | 2. The connection can be established + | + | Use this before switching to server mode to ensure the server is running. + +bd duplicate + use: "duplicate --of " + short: "Mark an issue as a duplicate of another" + group: "deps" + hidden: false + runnable: true + long: + | Mark an issue as a duplicate of a canonical issue. + | + | The duplicate issue is automatically closed with a reference to the canonical. + | This is essential for large issue databases with many similar reports. + | + | Examples: + | bd duplicate bd-abc --of bd-xyz # Mark bd-abc as duplicate of bd-xyz + flag --of type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Canonical issue ID (required)" + +bd duplicates + use: "duplicates" + short: "Find and optionally merge duplicate issues" + group: "deps" + hidden: false + runnable: true + long: + | Find issues with identical content (title, description, design, acceptance criteria). + | Groups issues by content hash and reports duplicates with suggested merge targets. + | The merge target is chosen by: + | 1. Reference count (most referenced issue wins) + | 2. Lexicographically smallest ID if reference counts are equal + | Only groups issues with matching status (open with open, closed with closed). + | Example: + | bd duplicates # Show all duplicate groups + | bd duplicates --auto-merge # Automatically merge all duplicates + | bd duplicates --dry-run # Show what would be merged + flag --auto-merge type=bool default="false" usage="Automatically merge all duplicates" + flag --dry-run type=bool default="false" usage="Show what would be merged without making changes" + +bd edit + use: "edit [id]" + short: "Edit an issue field in $EDITOR" + group: "issues" + hidden: false + runnable: true + long: + | Edit an issue field using your configured $EDITOR. + | + | By default, edits the description. Use flags to edit other fields. + | + | Examples: + | bd edit bd-42 # Edit description + | bd edit bd-42 --title # Edit title + | bd edit bd-42 --design # Edit design notes + | bd edit bd-42 --notes # Edit notes + | bd edit bd-42 --acceptance # Edit acceptance criteria + flag --acceptance type=bool default="false" usage="Edit the acceptance criteria" + flag --description type=bool default="false" usage="Edit the description (default)" + flag --design type=bool default="false" usage="Edit the design notes" + flag --notes type=bool default="false" usage="Edit the notes" + flag --title type=bool default="false" usage="Edit the title" + +bd epic + use: "epic" + short: "Epic management commands" + group: "deps" + hidden: false + runnable: false + +bd epic close-eligible + use: "close-eligible" + short: "Close epics where all children are complete" + hidden: false + runnable: true + flag --dry-run type=bool default="false" usage="Preview what would be closed without making changes" + flag --reason -r type=string default="All children completed" usage="Close reason applied to every epic closed" + +bd epic status + use: "status" + short: "Show epic completion status" + hidden: false + runnable: true + flag --eligible-only type=bool default="false" usage="Show only epics eligible for closure" + +bd events + use: "events" + short: "Read and manage the durable events journal" + group: "maint" + hidden: false + runnable: false + long: + | Read and manage the durable events journal (bd_events_journal). + | + | The journal records every committed issue mutation as an ordered, replayable + | row. Enable it with 'bd config set events-journal true' (or + | BD_EVENTS_JOURNAL=1). Records are emitted only while it is enabled. + | + | Retention is automatic: an enabled journal is bounded to the retention floors + | (events-journal-retain-days / -rows, 7 days / 100k rows by default) without + | anyone running a command. Disable both floors for an unbounded ledger, or + | events-journal-auto-prune for manual control; 'bd events prune' remains for an + | earlier, on-demand cut below the floors. + | + | Coverage and scope: + | - Every mutation through bd's normal write paths (create, update, close, + | reopen, delete, claim, dependency add/remove, label add/remove, comment) is + | journaled in the same transaction as the change. Raw DML run through + | 'bd sql' bypasses those paths and is NOT journaled — a known non-coverage. + | - The journal is per-branch working-set state (dolt_ignored): it records the + | mutations committed on the writer's active branch. Rows arrive by direct + | write, not by merge, so a consumer must read the journal on the same branch + | the writer commits to; a branch checkout or merge does not carry journal + | rows across branches. + | - For the same reason the journal is per REPLICA. 'bd dolt pull' and the + | changes a merge settles into this clone are not journaled: those rows + | arrived as data, not as local mutations, and nothing on this clone wrote + | them through the mutation seam. A consumer that mirrors a synced workspace + | must re-baseline (a fresh export or a full re-read) after a sync, because + | the journal describes only what THIS clone mutated. + | Each replica also has its OWN seq space, counted from its own first + | mutation. A checkpoint taken against one replica is meaningless against + | another — the same seq names a different record, and a seq above the other + | replica's head reads as "caught up" and stalls forever. Track a checkpoint + | per replica, and re-baseline rather than carry one across. + | - A few writes that happen while a store is being OPENED are unjournaled by + | design: schema migrations and the version reconciliation that runs before + | the workspace's configuration has been applied to the store. They touch + | schema and clone-local metadata, never a bead, so a replaying consumer has + | nothing to apply them to. + | - Dependency records are not symmetric, in two ways. + | Count: a dep_add is emitted for every accepted add, INCLUDING an idempotent + | same-type re-add that only refreshes the edge's metadata. The audit 'events' + | table deduplicates that case and writes nothing; the journal does not. Treat + | dep_add as an upsert of the edge, not as proof the edge is new. A dep_remove + | naming an edge that is already gone emits nothing at all. + | Payload: dep.metadata differs in provenance between the two ops. On dep_add + | it is the value being written, as the caller supplied it; on dep_remove it is + | the raw stored column read back just before the delete. The two can differ + | byte for byte while meaning the same thing, so compare parsed values. + | + | Structural dependency edits — the ones bd wires up itself rather than a 'bd dep' + | verb — write no audit event but DO journal, by design: a replaying consumer + | needs the edge either way. + +bd events export + use: "export" + short: "Print the entire journal from the beginning (JSON lines)" + hidden: false + runnable: true + long: + | Print every events journal record from seq 1, in order, as JSON lines. + | + | Equivalent to 'bd events tail --since 0'. Like tail, it FAILS rather than + | present a pruned journal's surviving suffix as a complete history. + flag --limit type=int default="0" usage="maximum number of records to return (0 = no limit)" + +bd events prune + use: "prune" + short: "Delete journal records below a sequence number (retention)" + hidden: false + runnable: true + long: + | Delete events journal records with seq less than --before. + | + | Retention is already enforced automatically: after a mutating command commits, + | and on a timer in 'bd serve', bd deletes everything the floors below do not + | protect. This command is for an EARLIER, on-demand cut BELOW the floors — after + | a consumer has durably processed a span you do not want to wait out. It cannot + | cut deeper than the floors: shrinking the retained window itself means lowering + | them. The journal is clone-local operational state, so pruning never affects + | issue data. + | + | Two retention floors compose onto --before and can only reduce what a prune + | removes. They bound the automatic prune and this one identically: + | events-journal-retain-days keep every row younger than N days (default 7) + | events-journal-retain-rows always keep the newest N rows (default 100000) + | + | Set BOTH floors to 0 for an unbounded ledger: automatic pruning then does + | nothing, and this command becomes the only thing that deletes a record. To keep + | the floors but own deletion yourself, set events-journal-auto-prune false. + | + | Note the floors are time-based and count-based — they are NOT a consumer + | watermark. They protect only the recent window; a consumer that has fallen + | further behind than both floors allow will be pruned past and lose records. + | Consumers are responsible for tracking their own watermark (the highest seq they + | have durably processed) and for sizing the floors to the longest outage they + | intend to survive. Pruned history cannot be recovered from the workspace — the + | journal is the only local copy. Pruning frees rows, not disk: pair it with + | 'dolt gc' to reclaim the space, since the table is working-set (dolt_ignored) + | state that ordinary Dolt commits never garbage-collect. + flag --before type=int64 default="0" usage="delete records with seq less than this value" + +bd events tail + use: "tail" + short: "Print journal records after a sequence number (JSON lines)" + hidden: false + runnable: true + long: + | Print events journal records with seq greater than --since, in order. + | + | Each line is a JSON record: + | {"seq":N,"ts":"...","op":"create|update|close|delete|dep_add|dep_remove|comment", + | "issue_id":"...","actor":"...","issue":{...|null},"dep":{"kind":..,"target":..,"metadata":..},"comment":{...}} + | + | Record contract (stable for external consumers): + | seq int64 counter-assigned inside the mutation's transaction; gapless, + | strictly increasing in commit order, never reused or reset + | ts string UTC insert time, stamped inside the committing transaction + | op string one of the seven ops above + | issue_id string the mutated issue's id + | actor string the acting identity that performed the mutation, as resolved + | for the audit-events table (on a comment row: the comment's + | author). A delete — and the dep_remove rows a cascading + | delete produces — carries the identity that REQUESTED it. + | Empty (omitted) only when the path genuinely has no actor: + | derived maintenance, system cleanup with no request behind + | it, and rows older than the column. Never user attribution + | when empty. + | issue object full issue state AFTER the mutation; null on delete + | dep object {"kind","target","metadata"} for dep_add / dep_remove; omitted otherwise + | comment object {"id","author","text","created_at","source"} for comment; omitted otherwise + | + | Poll with the highest seq seen to consume new mutations incrementally, or pass + | --follow to keep printing new records as they are committed (Ctrl-C to stop). + | + | Retention boundary: if --since is below the oldest retained record — the prefix + | you asked for was pruned — the read FAILS instead of silently skipping ahead or + | returning an empty success. With --json the failure carries + | {"code":"events_journal_truncated","since":N,"floor":F,"head":H}: floor is the + | oldest seq still retained, head the highest ever assigned. Resume from floor-1 + | to continue with a known gap, or rebuild from a full export. + flag --follow type=bool default="false" usage="keep printing new records as they are committed (Ctrl-C to stop)" + flag --limit type=int default="0" usage="maximum number of records to return (0 = no limit)" + flag --since type=int64 default="0" usage="return records with seq greater than this value" + +bd export + use: "export" + short: "Export issues to JSONL format" + group: "sync" + hidden: false + runnable: true + long: + | Export all issues to JSONL (newline-delimited JSON) format. + | + | Each line is a complete JSON object representing one issue, including its + | labels, dependencies, and comments. + | + | This command is for issue export, migration, and interoperability. It exports + | records from the issues table; it is not a full database backup and does not + | capture Dolt branches, commit history, working-set state, or non-issue tables. + | For supported full backup/restore flows, use 'bd backup init', 'bd backup sync', + | and 'bd backup restore'. + | + | By default, exports only regular issues (excluding infrastructure beads + | like agents, roles, and messages). Use --all to include everything. + | + | Memories (from 'bd remember') are excluded by default because they may + | contain sensitive agent context. Use --include-memories or --all to + | include them. + | + | EXAMPLES: + | bd export # Export issues to stdout + | bd export -o issues.jsonl # Export issues to file + | bd export --include-memories # Export issues + memories + | bd export --all -o full.jsonl # Include infra + templates + gates + memories + | bd export --scrub -o clean.jsonl # Exclude test/pollution records + flag --all type=bool default="false" usage="Include all records (infra, templates, gates, memories)" + flag --exclude-owner type=stringArray default="[]" usage="Exclude issues created by this identity (repeatable; also reads export.exclude_owners config)" + flag --include-infra type=bool default="false" usage="Include infrastructure beads (agents, roles, messages)" + flag --include-memories type=bool default="false" usage="Include persistent memories (from 'bd remember') in the export" + flag --no-memories type=bool default="false" hidden usage="Exclude persistent memories (deprecated: now the default)" + flag --output -o type=string default="" usage="Output file path (default: stdout)" + flag --scrub type=bool default="false" usage="Exclude test/pollution records" + flag --verbose type=bool default="false" usage="Print filtered issue count when owners are excluded" + +bd federation + use: "federation" + short: "Manage peer-to-peer federation with other workspaces" + group: "sync" + hidden: false + runnable: false + long: + | Manage peer-to-peer federation between Dolt-backed beads databases. + | + | Federation enables synchronized issue tracking across multiple workspaces, + | each maintaining their own Dolt database while sharing updates via remotes. + | + | Requires the Dolt storage backend. + +bd federation add-peer + use: "add-peer " + short: "Add a federation peer with optional SQL credentials" + hidden: false + runnable: true + long: + | Add a new federation peer remote with optional SQL user authentication. + | + | The URL can be: + | - dolthub://org/repo DoltHub hosted repository + | - host:port/database Direct dolt sql-server connection + | - file:///path/to/repo Local file path (for testing) + | + | Credentials are encrypted and stored locally. They are used automatically + | when syncing with the peer. If --user is provided without --password, + | you will be prompted for the password interactively. + | + | Examples: + | bd federation add-peer town-beta dolthub://acme/town-beta-beads + | bd federation add-peer town-gamma 192.168.1.100:3306/beads --user sync-bot + | bd federation add-peer partner https://partner.example.com/beads --user admin --password secret + flag --password -p type=string default="" usage="SQL password (prompted if --user set without --password)" + flag --sovereignty type=string default="" usage="Sovereignty tier (T1, T2, T3, T4)" + flag --user -u type=string default="" usage="SQL username for authentication" + +bd federation list-peers + use: "list-peers" + short: "List configured federation peers" + hidden: false + runnable: true + +bd federation remove-peer + use: "remove-peer " + short: "Remove a federation peer" + hidden: false + runnable: true + +bd federation status + use: "status [--peer name]" + short: "Show federation sync status" + hidden: false + runnable: true + long: + | Show synchronization status with peer towns. + | + | Displays: + | - Configured peers and their URLs + | - Commits ahead/behind each peer + | - Whether there are unresolved conflicts + | + | Examples: + | bd federation status # Status for all peers + | bd federation status --peer town-beta # Status for specific peer + flag --peer type=string default="" usage="Specific peer to check" + +bd federation sync + use: "sync [--peer name]" + short: "Synchronize with a peer town" + hidden: false + runnable: true + long: + | Pull from and push to peer towns. + | + | Without --peer, syncs with all configured peers. + | With --peer, syncs only with the specified peer. + | + | Handles merge conflicts using the configured strategy: + | --strategy ours Keep local changes on conflict + | --strategy theirs Accept remote changes on conflict + | + | If no strategy is specified and conflicts occur, the sync will pause + | and report which tables have conflicts for manual resolution. + | + | Examples: + | bd federation sync # Sync with all peers + | bd federation sync --peer town-beta # Sync with specific peer + | bd federation sync --strategy theirs # Auto-resolve using remote values + flag --peer type=string default="" usage="Specific peer to sync with" + flag --strategy type=string default="" usage="Conflict resolution strategy (ours|theirs)" + +bd find-duplicates + use: "find-duplicates" + aliases: ["find-dups"] + short: "Find semantically similar issues using text analysis or AI" + group: "views" + hidden: false + runnable: true + long: + | Find issues that are semantically similar but not exact duplicates. + | + | Unlike 'bd duplicates' which finds exact content matches, find-duplicates + | uses text similarity or AI to find issues that discuss the same topic + | with different wording. + | + | Approaches: + | mechanical Token-based text similarity (default, no API key needed) + | ai LLM-based semantic comparison (requires ANTHROPIC_API_KEY, MINIMAX_API_KEY, or ai.api_key) + | + | The mechanical approach tokenizes titles and descriptions, then computes + | Jaccard similarity between all issue pairs. It's fast and free but may + | miss semantically similar issues with very different wording. + | + | The AI approach sends candidate pairs to an Anthropic-compatible model for semantic comparison. + | It first uses mechanical pre-filtering to reduce the number of API calls, + | then asks the LLM to judge whether the remaining pairs are true duplicates. + | + | Examples: + | bd find-duplicates # Mechanical similarity (default) + | bd find-duplicates --threshold 0.4 # Lower threshold = more results + | bd find-duplicates --method ai # Use AI for semantic comparison + | bd find-duplicates --status open # Only check open issues + | bd find-duplicates --limit 20 # Show top 20 pairs + | bd find-duplicates --json # JSON output + flag --limit -n type=int default="50" usage="Maximum number of pairs to show" + flag --max-rows type=int default="0" usage="Hard upper bound on rows fetched from storage. Returns a non-zero exit (code 2) and an error to stderr if exceeded. 0 disables (the default). Overrides BEADS_MAX_ROWS for this invocation. Useful in CI/agent rigs that want a circuit breaker against pathological queries. Not supported under --proxied-server: an explicit --max-rows or BEADS_MAX_ROWS cap errors out rather than silently going unenforced." + flag --method type=string default="mechanical" usage="Detection method: mechanical, ai" + flag --model type=string default="" usage="AI model to use (only with --method ai; default from config ai.model)" + flag --status -s type=string default="" usage="Filter by status (default: non-closed)" + flag --threshold type=float64 default="0.5" usage="Similarity threshold (0.0-1.0, lower = more results)" + +bd flatten + use: "flatten" + short: "Squash all Dolt history into a single commit" + group: "maint" + hidden: false + runnable: true + long: + | Nuclear option: squash ALL Dolt commit history into a single commit. + | + | This uses the Tim Sehn recipe: + | 1. Create a new branch from the current state + | 2. Soft-reset to the initial commit (preserving all data) + | 3. Commit everything as a single snapshot + | 4. Swap main branch to the new flattened branch + | 5. Prune remote-tracking refs (they would keep the old history alive; + | the next push or fetch re-creates them at the new tip) + | 6. Run a full Dolt GC (all storage generations) to reclaim the old history + | + | The GC pass is a full collection: Dolt storage is generational, and a default + | GC never revisits data an earlier GC moved to the old generation. On any store + | that has been GC'd before (bd gc, or a previous flatten or compact), only a + | full collection reclaims the squashed history. A full GC can take minutes on + | multi-gigabyte stores. + | + | This is irreversible — all commit history is lost. The resulting database + | has exactly one commit containing all current data. + | + | Use this when: + | - Your active database has grown very large + | - You don't need commit-level history (time travel) + | - You want to start fresh with minimal storage + | + | Examples: + | bd flatten --dry-run # Preview: show commit count and disk usage + | bd flatten --force # Actually squash all history + | bd flatten --force --json # JSON output + flag --dry-run type=bool default="false" usage="Preview without making changes" + flag --force -f type=bool default="false" usage="Confirm irreversible history squash" + +bd forget + use: "forget " + short: "Remove a persistent memory" + group: "setup" + hidden: false + runnable: true + long: + | Remove a memory by its key. + | + | Use 'bd memories' to see available keys. + | + | Examples: + | bd forget dolt-phantoms + | bd forget auth-jwt + +bd formula + use: "formula" + short: "Manage workflow formulas" + hidden: false + runnable: false + long: + | Manage workflow formulas - the source layer for molecule templates. + | + | Formulas are TOML/JSON files that define workflows with composition rules. + | Define formulas, cook them into protos, then pour or wisp them into work. + | + | Search paths (in order): + | 1. /formulas/ (active project) + | 2. /.beads/formulas/ (repo-local formulas) + | 3. ~/.beads/formulas/ (user) + | 4. $GT_ROOT/.beads/formulas/ (shared workspace root, if GT_ROOT set) + | + | Commands: + | list List available formulas from all search paths + | show Show formula details, steps, and composition rules + | schema Show the formula schema index (alias: primitives) + | + | Discovering primitives: + | bd formula schema # list every declared formula struct + | bd formula schema loop # show LoopSpec fields, types, and tags + | bd formula primitives gate # alias; same handler as 'schema' + | examples/formulas/primitives/ # curated, smoke-tested wired fixtures + | docs/workflows/formulas.md # narrative reference + +bd formula convert + use: "convert [--all]" + short: "Convert formula from JSON to TOML" + hidden: false + runnable: true + long: + | Convert formula files from JSON to TOML format. + | + | TOML format provides better ergonomics: + | - Multi-line strings without \n escaping + | - Human-readable diffs + | - Comments allowed + | + | The convert command reads a .formula.json file and outputs .formula.toml. + | The original JSON file is preserved (use --delete to remove it). + | + | Examples: + | bd formula convert shiny # Convert shiny.formula.json to .toml + | bd formula convert ./my.formula.json # Convert specific file + | bd formula convert --all # Convert all JSON formulas + | bd formula convert shiny --delete # Convert and remove JSON file + | bd formula convert shiny --stdout # Print TOML to stdout + flag --all type=bool default="false" usage="Convert all JSON formulas" + flag --delete type=bool default="false" usage="Delete JSON file after conversion" + flag --stdout type=bool default="false" usage="Print TOML to stdout instead of file" + +bd formula list + use: "list" + short: "List available formulas" + hidden: false + runnable: true + long: + | List all formulas from search paths. + | + | Search paths (in order of priority): + | 1. /formulas/ (active project - highest priority) + | 2. /.beads/formulas/ (repo-local formulas) + | 3. ~/.beads/formulas/ (user) + | 4. $GT_ROOT/.beads/formulas/ (shared workspace root, if GT_ROOT set) + | + | Formulas in earlier paths shadow those with the same name in later paths. + | + | To list the declared formula schema structs an agent can write inside a .formula.toml, + | use 'bd formula schema' (alias: 'bd formula primitives'). + | + | Examples: + | bd formula list + | bd formula list --json + | bd formula list --type workflow + | bd formula list --type convoy + flag --type type=string default="" usage="Filter by type (workflow, expansion, aspect, convoy)" + +bd formula schema + use: "schema [primitive]" + aliases: ["primitives"] + short: "Show the formula schema index (every exported struct in types.go)" + hidden: false + runnable: true + long: + | Show the formula schema index: every exported struct declared + | in a .formula.toml/.formula.json, with field names, types, and tags. + | + | The index is generated from internal/formula/types.go via go:generate; the + | struct definitions are the source of truth, so this list cannot drift. It is + | structural reference, not proof that every declared runtime behavior is wired. + | + | Examples: + | bd formula schema # list every declared schema struct + | bd formula schema loop # show LoopSpec fields + | bd formula primitives gate # alias; shows Gate fields + | bd formula schema --json # machine-readable index + | + | Curated smoke-tested fixtures for wired primitives live in + | examples/formulas/primitives/ (with a smoke harness that proves they work). + +bd formula show + use: "show " + short: "Show formula details" + hidden: false + runnable: true + long: + | Show detailed information about a formula. + | + | Displays: + | - Formula metadata (name, type, description) + | - Variables with defaults and constraints + | - Steps with dependencies + | - Composition rules (extends, aspects, expansions) + | - Bond points for external composition + | + | To inspect the structure of an individual primitive (e.g. LoopSpec, Gate) + | rather than a user-authored formula, use 'bd formula schema '. + | + | Examples: + | bd formula show shiny + | bd formula show rule-of-five + | bd formula show security-audit --json + +bd gate + use: "gate" + short: "Manage async coordination gates" + group: "issues" + hidden: false + runnable: false + long: + | Gates are async wait conditions that block workflow steps. + | + | Gates are created automatically when a formula step has a gate field. + | They must be closed (manually or via watchers) for the blocked step to proceed. + | + | Gate types: + | human - Requires manual bd close (Phase 1) + | timer - Expires after timeout (Phase 2) + | gh:run - Waits for GitHub workflow (Phase 3) + | gh:pr - Waits for PR merge (Phase 3) + | bead - Waits for another bead to close (Phase 4) + | + | For bead gates, await_id may be a bead ID in this rig's database (e.g., + | "bd-abc123") or the historical cross-rig form :. Cross-rig + | targets resolve through the bead ID's prefix route in routes.jsonl. + | + | Examples: + | bd gate list # Show all open gates + | bd gate list --all # Show all gates including closed + | bd gate check # Evaluate all open gates + | bd gate check --type=bead # Evaluate only bead gates + | bd gate resolve # Close a gate manually + +bd gate add-waiter + use: "add-waiter " + short: "Add a waiter to a gate" + hidden: false + runnable: true + long: + | Register an agent as a waiter on a gate bead. + | + | When the gate closes, the waiter will receive a wake notification via 'bd gate wake'. + | The waiter is typically the worker's address (e.g., "my-project/workers/agent-1"). + | + | This is used by 'bd done --phase-complete' to register for gate wake notifications. + +bd gate check + use: "check" + short: "Evaluate gates and close resolved ones" + hidden: false + runnable: true + long: + | Evaluate gate conditions and automatically close resolved gates. + | + | By default, checks all open gates. Use --type to filter by gate type. + | + | Gate types: + | gh - Check all GitHub gates (gh:run and gh:pr) + | gh:run - Check GitHub Actions workflow runs + | gh:pr - Check pull request merge status + | timer - Check timer gates (auto-expire based on timeout) + | bead - Check bead gates + | all - Check all gate types + | + | GitHub gates use the 'gh' CLI to query status: + | - gh:run checks 'gh run view --json status,conclusion' + | - gh:pr checks 'gh pr view --json state,title' + | + | A gate is resolved when: + | - gh:run: status=completed AND conclusion=success + | - gh:pr: state=MERGED + | - timer: current time > created_at + timeout + | - bead: target bead status=closed, or a bead an earlier check saw in + | this rig no longer exists + | + | A gate is escalated when: + | - gh:run: status=completed AND conclusion in (failure, canceled) + | - gh:pr: state=CLOSED + | + | Examples: + | bd gate check # Check all gates + | bd gate check --type=gh # Check only GitHub gates + | bd gate check --type=gh:run # Check only workflow run gates + | bd gate check --type=timer # Check only timer gates + | bd gate check --type=bead # Check only bead gates + | bd gate check --dry-run # Show what would happen without changes + | bd gate check --escalate # Escalate expired/failed gates + flag --dry-run type=bool default="false" usage="Show what would happen without making changes" + flag --escalate -e type=bool default="false" usage="Escalate failed/expired gates" + flag --limit -l type=int default="100" usage="Limit results (default 100)" + flag --type -t type=string default="" usage="Gate type to check (gh, gh:run, gh:pr, timer, bead, all)" + +bd gate create + use: "create" + short: "Create a gate that blocks an issue" + hidden: false + runnable: true + long: + | Create an ad-hoc gate issue that blocks another issue until resolved. + | + | The blocked issue will not appear in 'bd ready' until the gate is resolved + | via 'bd gate resolve'. + | + | Gate types: + | human - Requires manual 'bd gate resolve' (default) + | timer - Auto-resolves after --timeout duration + | gh:run - Waits for GitHub Actions workflow + | gh:pr - Waits for PR merge + | + | gh:run and gh:pr gates are checked in the current Git repository unless + | --repo names another, or the blocked issue carries a metadata.repo value. + | + | Examples: + | bd gate create --blocks bd-abc + | bd gate create --type=human --blocks bd-abc --reason="Need design review" + | bd gate create --type=timer --blocks bd-abc --timeout=2h + | bd gate create --type=gh:pr --blocks bd-abc --await-id=42 + | bd gate create --type=gh:pr --blocks bd-abc --await-id=42 --repo=owner/other-repo + | bd gate create --blocks bd-abc --title="Gate: awaiting owner sign-off" + flag --await-id type=string default="" usage="Condition identifier (run ID, PR number, etc.)" + flag --blocks type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Issue ID to block (required)" + flag --reason -r type=string default="" usage="Reason for the gate" + flag --repo type=string default="" usage="GitHub repository the gh:run/gh:pr condition is checked in (OWNER/REPO or HOST/OWNER/REPO); default: the blocked issue's metadata.repo, else the current repository" + flag --timeout type=string default="" usage="Timeout duration (e.g., 2h, 30m)" + flag --title type=string default="" usage="Custom gate title (default: \"Gate: \")" + flag --type -t type=string default="human" usage="Gate type (human, timer, gh:run, gh:pr)" + +bd gate discover + use: "discover" + short: "Discover await_id for gh:run gates" + hidden: false + runnable: true + long: + | Discovers GitHub workflow run IDs for gates awaiting CI/CD completion. + | + | This command finds open gates with await_type="gh:run" that don't have an await_id, + | queries recent GitHub workflow runs, and matches them using heuristics: + | - Branch name matching + | - Commit SHA matching + | - Time proximity (runs within 5 minutes of gate creation) + | + | Once matched, the gate's await_id is updated with the GitHub run ID, enabling + | subsequent polling to check the run's status. + | + | A gate whose metadata.repo targets another repository is only matched + | against runs queried from that repository, never against the current + | repository's runs of a same-named workflow. + | + | Examples: + | bd gate discover # Auto-discover run IDs for all matching gates + | bd gate discover --dry-run # Preview what would be matched (no updates) + | bd gate discover --branch main --limit 10 # Only match runs on 'main' branch + flag --branch -b type=string default="" usage="Filter runs by branch (default: current branch)" + flag --dry-run -n type=bool default="false" usage="Preview mode: show matches without updating" + flag --limit -l type=int default="10" usage="Max runs to query from GitHub" + flag --max-age -a type=duration default="30m0s" usage="Max age for gate/run matching" + +bd gate list + use: "list [issue-id]" + short: "List gate issues" + hidden: false + runnable: true + long: + | List gate issues. + | + | With no argument, lists all gate issues in the current beads database. + | With an [issue-id] argument, lists ONLY the gates that block that issue + | (its own dependency gates) — not every gate in the database. + | + | By default, shows only open gates. Use --all to include closed gates. + flag --all -a type=bool default="false" usage="Show all gates including closed" + flag --limit -n type=int default="50" usage="Limit results (default 50)" + +bd gate resolve + use: "resolve " + short: "Manually resolve (close) a gate" + hidden: false + runnable: true + long: + | Close a gate issue to unblock the step waiting on it. + | + | This is equivalent to 'bd close ' but with a more explicit name. + | Use --reason to provide context for why the gate was resolved. + flag --reason -r type=string default="" usage="Reason for resolving the gate" + +bd gate show + use: "show " + short: "Show a gate issue" + hidden: false + runnable: true + long: + | Display details of a gate issue including its waiters. + | + | This is similar to 'bd show' but validates that the issue is a gate. + +bd gc + use: "gc" + short: "Garbage collect: decay old issues, compact Dolt commits, run Dolt GC" + group: "maint" + hidden: false + runnable: true + long: + | Full lifecycle garbage collection for standalone Beads databases. + | + | Runs three phases in sequence: + | 1. DECAY — Delete closed issues older than N days (default 90) + | 2. COMPACT — Squash old Dolt commits into fewer commits (bd compact) + | 3. GC — Run Dolt garbage collection to reclaim disk space + | + | Each phase can be skipped individually. Use --dry-run to preview all phases + | without making changes. + | + | Phase 3 runs Dolt's default, generational GC: it only examines data written + | since the last GC. Data that survived an earlier GC lives in the old generation + | and is never revisited, so on a long-lived store the space freed by decay may + | not be reclaimed. Use --full to collect all generations (slower on large + | stores). + | + | Examples: + | bd gc # Full GC with defaults (90 day decay) + | bd gc --dry-run # Preview what would happen + | bd gc --older-than 30 # Decay issues closed 30+ days ago + | bd gc --skip-decay # Skip issue deletion, just compact+GC + | bd gc --skip-dolt # Skip Dolt GC, just decay+compact + | bd gc --full # Collect all storage generations + | bd gc --force # Skip confirmation prompt + flag --dry-run type=bool default="false" usage="Preview without making changes" + flag --force -f type=bool default="false" usage="Skip confirmation prompts" + flag --full type=bool default="false" usage="Run a full Dolt GC (all generations; slower, reclaims space default passes cannot)" + flag --older-than type=int default="90" usage="Delete closed issues older than N days" + flag --skip-decay type=bool default="false" usage="Skip issue deletion phase" + flag --skip-dolt type=bool default="false" usage="Skip Dolt garbage collection phase" + +bd github + use: "github" + short: "GitHub integration commands" + hidden: false + runnable: false + long: + | Commands for syncing issues between beads and GitHub. + | + | Configuration can be set via 'bd config' or environment variables: + | github.token / GITHUB_TOKEN - Personal access token + | github.owner / GITHUB_OWNER - Repository owner + | github.repo / GITHUB_REPO - Repository name + | github.repository / GITHUB_REPOSITORY - Combined "owner/repo" format + | github.url / GITHUB_API_URL - Custom API URL (GitHub Enterprise) + +bd github pull + use: "pull [refs...]" + short: "Pull specific items from GitHub" + hidden: false + runnable: true + long: + | Pull one or more items from GitHub. + | + | Accepts bead IDs or external references as positional arguments. + | Equivalent to: bd github sync --pull-only --issues + flag --dry-run type=bool default="false" usage="Preview pull without making changes" + +bd github push + use: "push [bead-ids...]" + short: "Push specific beads to GitHub" + hidden: false + runnable: true + long: + | Push one or more beads issues to GitHub. + | + | Accepts bead IDs as positional arguments. + | Equivalent to: bd github sync --push-only --issues + flag --dry-run type=bool default="false" usage="Preview push without making changes" + +bd github repos + use: "repos" + short: "List accessible GitHub repositories" + hidden: false + runnable: true + long: + | List GitHub repositories that the configured token has access to. + +bd github status + use: "status" + short: "Show GitHub sync status" + hidden: false + runnable: true + long: + | Display current GitHub configuration and sync status. + +bd github sync + use: "sync" + short: "Sync issues with GitHub" + hidden: false + runnable: true + long: + | Synchronize issues between beads and GitHub. + | + | By default, performs bidirectional sync: + | - Pulls new/updated issues from GitHub to beads + | - Pushes local beads issues to GitHub + | + | Use --pull-only or --push-only to limit direction. + flag --dry-run type=bool default="false" usage="Show what would be synced without making changes" + flag --issues type=string default="" usage="Comma-separated bead IDs to sync selectively (e.g., bd-abc,bd-def). Mutually exclusive with --parent." + flag --parent type=string default="" usage="Limit push to this bead and its descendants (push only). Mutually exclusive with --issues." + flag --prefer-github type=bool default="false" usage="On conflict, use GitHub version" + flag --prefer-local type=bool default="false" usage="On conflict, keep local beads version" + flag --prefer-newer type=bool default="false" usage="On conflict, use most recent version (default)" + flag --pull-only type=bool default="false" usage="Only pull issues from GitHub" + flag --push-only type=bool default="false" usage="Only push issues to GitHub" + +bd gitlab + use: "gitlab" + short: "GitLab integration commands" + hidden: false + runnable: false + long: + | Commands for syncing issues between beads and GitLab. + | + | Configuration can be set via 'bd config' or environment variables: + | gitlab.url / GITLAB_URL - GitLab instance URL + | gitlab.token / GITLAB_TOKEN - Personal access token + | gitlab.project_id / GITLAB_PROJECT_ID - Project ID or path + | gitlab.group_id / GITLAB_GROUP_ID - Group ID for group-level sync + | gitlab.default_project_id / GITLAB_DEFAULT_PROJECT_ID - Project for creating issues in group mode + +bd gitlab projects + use: "projects" + short: "List accessible GitLab projects" + hidden: false + runnable: true + long: + | List GitLab projects that the configured token has access to. + +bd gitlab pull + use: "pull [refs...]" + short: "Pull specific items from GitLab" + hidden: false + runnable: true + long: + | Pull one or more items from GitLab. + | + | Accepts bead IDs or external references as positional arguments. + | Equivalent to: bd gitlab sync --pull-only --issues + flag --dry-run type=bool default="false" usage="Preview pull without making changes" + +bd gitlab push + use: "push [bead-ids...]" + short: "Push specific beads to GitLab" + hidden: false + runnable: true + long: + | Push one or more beads issues to GitLab. + | + | Accepts bead IDs as positional arguments. + | Equivalent to: bd gitlab sync --push-only --issues + flag --dry-run type=bool default="false" usage="Preview push without making changes" + +bd gitlab status + use: "status" + short: "Show GitLab sync status" + hidden: false + runnable: true + long: + | Display current GitLab configuration and sync status. + +bd gitlab sync + use: "sync" + short: "Sync issues with GitLab" + hidden: false + runnable: true + long: + | Synchronize issues between beads and GitLab. + | + | By default, performs bidirectional sync: + | - Pulls new/updated issues from GitLab to beads + | - Pushes local beads issues to GitLab + | + | Use --pull-only or --push-only to limit direction. + flag --assignee type=string default="" usage="Filter by assignee username" + flag --dry-run type=bool default="false" usage="Show what would be synced without making changes" + flag --exclude-type type=string default="" usage="Exclude these issue types from sync (comma-separated)" + flag --issues type=string default="" usage="Comma-separated bead IDs to sync selectively (e.g., bd-abc,bd-def). Mutually exclusive with --parent." + flag --label type=string default="" usage="Filter by labels (comma-separated, AND logic)" + flag --milestone type=string default="" usage="Filter by milestone title" + flag --no-ephemeral type=bool default="true" usage="Exclude ephemeral/wisp issues from push (default: true)" + flag --parent type=string default="" usage="Limit push to this bead and its descendants (push only). Mutually exclusive with --issues." + flag --prefer-gitlab type=bool default="false" usage="On conflict, use GitLab version" + flag --prefer-local type=bool default="false" usage="On conflict, keep local beads version" + flag --prefer-newer type=bool default="false" usage="On conflict, use most recent version (default)" + flag --project type=string default="" usage="Filter to issues from this project ID (group mode)" + flag --pull-only type=bool default="false" usage="Only pull issues from GitLab" + flag --push-only type=bool default="false" usage="Only push issues to GitLab" + flag --type type=string default="" usage="Only sync these issue types (comma-separated, e.g. 'epic,feature,task')" + +bd graph + use: "graph [issue-id]" + short: "Display issue dependency graph" + group: "deps" + hidden: false + runnable: true + long: + | Display a visualization of an issue's dependency graph. + | + | For epics, shows all children and their dependencies. + | For regular issues, shows the issue and its direct dependencies. + | + | With --all, shows all open issues grouped by connected component. + | With --open, filters to only open/actionable issues (compact layer format). + | + | Display formats: + | (default) DAG with columns and box-drawing edges (terminal-native) + | --box ASCII boxes showing layers, more detailed + | --compact Tree format, one line per issue, more scannable + | --dot Graphviz DOT format (pipe to dot -Tsvg > graph.svg) + | --html Self-contained interactive HTML with D3.js visualization + | --open Open issues only, compact layers (LLM-friendly) + | + | The graph shows execution order: + | - Layer 0 / leftmost = no dependencies (can start immediately) + | - Higher layers depend on lower layers + | - Nodes in the same layer can run in parallel + | + | Status icons: ○ open ◐ in_progress ● blocked ✓ closed ❄ deferred + | + | Examples: + | bd graph issue-id # Terminal DAG visualization (default) + | bd graph --box issue-id # ASCII boxes with layer grouping + | bd graph --dot issue-id | dot -Tsvg > graph.svg # SVG via Graphviz + | bd graph --dot issue-id | dot -Tpng > graph.png # PNG via Graphviz + | bd graph --html issue-id > graph.html # Interactive browser view + | bd graph --all --html > all.html # All issues, interactive + | bd graph --open issue-id # Open issues only, layered by blocking order + | bd graph --all --open # All open issues, compact layers + | + | --max-rows / BEADS_MAX_ROWS caveat: the cap is checked differently per mode. + | Single-issue graphs (no --all) check the connected-component node count + | after the BFS traversal completes — the whole subgraph is always walked + | first, then rejected if it's over cap. --all checks each status + | (open/in_progress/blocked) independently, so up to 3x the cap can be loaded + | in total before any individual status trips it. + flag --all type=bool default="false" usage="Show graph for all open issues" + flag --box type=bool default="false" usage="ASCII boxes showing layers" + flag --compact type=bool default="false" usage="Tree format, one line per issue, more scannable" + flag --dot type=bool default="false" usage="Output Graphviz DOT format (pipe to: dot -Tsvg > graph.svg)" + flag --html type=bool default="false" usage="Output self-contained interactive HTML (redirect to file)" + flag --max-rows type=int default="0" usage="Hard upper bound on rows fetched from storage. Returns a non-zero exit (code 2) and an error to stderr if exceeded. 0 disables (the default). Overrides BEADS_MAX_ROWS for this invocation. Useful in CI/agent rigs that want a circuit breaker against pathological queries. Not supported under --proxied-server: an explicit --max-rows or BEADS_MAX_ROWS cap errors out rather than silently going unenforced." + flag --open type=bool default="false" usage="Show only open issues (filters out closed/deferred), forces compact layer format" + +bd graph check + use: "check" + short: "Check dependency graph integrity" + hidden: false + runnable: true + long: + | Check the dependency graph for cycles, orphans, and other integrity issues. + | + | Returns exit code 0 if the graph is clean, 1 if issues are found. + +bd heartbeat + use: "heartbeat " + aliases: ["hb"] + short: "Refresh the lease on an issue you hold in_progress" + group: "issues" + hidden: false + runnable: true + long: + | Refresh the lease on an issue you currently hold in_progress. + | + | A claim carries a lease that expires after a TTL. A worker keeps its claim alive + | by heartbeating faster than the TTL; once it stops (because it died), the lease + | goes stale and 'bd reclaim' reverts the issue to ready so another worker can pick + | it up. Heartbeat pushes lease_expires_at forward and stamps heartbeat_at = now. + | + | Only the current owner may heartbeat. If the lease has already been reclaimed or + | the issue closed, heartbeat fails so the worker learns to stop. + | + | Leases live in an ephemeral, node-local table: a heartbeat that refreshes your + | lease writes no Dolt commit and no history, so any cadence comfortably below + | the TTL is fine. Only a re-arm (the first heartbeat of a claim that has no + | lease, such as one set by a plain update) is recorded as a change to the issue + | and committed, once. Leases are only enforceable on the node that granted them; + | cross-machine claim visibility rides the issue's status and assignee, which do + | commit. + | + | Examples: + | bd heartbeat bd-123 + | bd hb bd-123 + +bd help + use: "help [command]" + short: "Help about any command" + hidden: false + runnable: true + long: + | Help provides help for any command in the application. + | Simply type bd help [path to command] for full details. + flag --all type=bool default="false" usage="Show help for all commands in a single document" + flag --doc type=string default="" usage="Generate markdown docs for a single command" + flag --docs-root type=string default="" usage="Generate repository CLI docs under this root" + flag --list type=bool default="false" usage="List all available commands" + +bd history + use: "history " + short: "Show version history for an issue" + group: "views" + hidden: false + runnable: true + long: + | Show the complete version history of an issue, including all commits + | where the issue was modified. + | + | Examples: + | bd history bd-123 # Show all history for issue bd-123 + | bd history bd-123 --limit 5 # Show last 5 changes + | bd history bd-123 --events # Show database audit events + flag --events type=bool default="false" usage="Show database audit events instead of commit snapshots" + flag --limit type=int default="0" usage="Limit number of history entries (0 = all)" + +bd hooks + use: "hooks" + short: "Manage git hooks for beads integration" + group: "setup" + hidden: false + runnable: false + long: + | Install, uninstall, or list git hooks for beads integration. + | + | The hooks provide: + | - pre-commit: Run chained hooks before commit + | - post-merge: Run chained hooks after pull/merge + | - pre-push: Run chained hooks before push + | - post-checkout: Run chained hooks after branch checkout + | - prepare-commit-msg: Add agent identity trailers for forensics + +bd hooks install + use: "install" + short: "Install bd git hooks" + hidden: false + runnable: true + long: + | Install git hooks for beads integration. + | + | By default, hooks are installed to .git/hooks/ in the current repository. + | Use --beads to install to .beads/hooks/ (recommended for Dolt backend). + | Use --shared to install to a versioned directory (.beads-hooks/) that can be + | committed to git and shared with team members. + | + | Hooks use section markers to coexist with existing hooks — any user content + | outside the markers is preserved across installs and upgrades. + | + | Installed hooks: + | - pre-commit: Run chained hooks before commit + | - post-merge: Run chained hooks after pull/merge + | - pre-push: Run chained hooks before push + | - post-checkout: Run chained hooks after branch checkout + | - prepare-commit-msg: Add agent identity trailers (for orchestrator agents) + flag --beads type=bool default="false" usage="Install hooks to .beads/hooks/ (recommended for Dolt backend)" + flag --chain type=bool default="false" usage="No-op, kept for compatibility (existing hook content always runs alongside the bd section)" + flag --force type=bool default="false" usage="No-op, kept for compatibility (section markers always preserve non-bd content)" + flag --shared type=bool default="false" usage="Install hooks to .beads-hooks/ (versioned) instead of .git/hooks/" + +bd hooks list + use: "list" + short: "List installed git hooks status" + hidden: false + runnable: true + long: + | Show the status of bd git hooks (installed, outdated, missing). + +bd hooks run + use: "run [args...]" + short: "Execute a git hook (called by thin shims)" + hidden: false + runnable: true + long: + | Execute the logic for a git hook. This command is typically called by + | thin shim scripts installed in .git/hooks/. + | + | Supported hooks: + | - pre-commit: Run chained hooks before commit + | - post-merge: Run chained hooks after pull/merge + | - pre-push: Run chained hooks before push + | - post-checkout: Run chained hooks after branch checkout + | - prepare-commit-msg: Add agent identity trailers for forensics + | + | The thin shim keeps delegated hook logic in sync with the installed bd + | version. Upgrading bd updates that delegated behavior. To adopt changes to the + | shim's generated shell policy, refresh it with 'bd hooks install'. + +bd hooks uninstall + use: "uninstall" + short: "Uninstall bd git hooks" + hidden: false + runnable: true + long: + | Remove bd git hooks from .git/hooks/ directory. + +bd human + use: "human" + short: "Show essential commands for human users" + group: "setup" + hidden: false + runnable: true + long: + | Display a focused help menu showing only the most common commands. + | + | bd has 70+ commands - many for AI agents, integrations, and advanced workflows. + | This command shows the ~15 essential commands that human users need most often. + | + | For the full command list, run: bd --help + | + | SUBCOMMANDS: + | human list List human-needed beads (issues with 'human' label; hides closed by default) + | human respond Respond to a human-needed bead (adds comment and closes) + | human dismiss Dismiss a human-needed bead permanently + | human stats Show summary statistics for human-needed beads + +bd human dismiss + use: "dismiss [reason...]" + short: "Dismiss a human-needed bead" + hidden: false + runnable: true + long: + | Dismiss a human-needed bead permanently without responding. + | + | The issue is closed with a "Dismissed" reason and optional note. + | The reason can be given as positional arguments or --reason. + | + | Examples: + | bd human dismiss bd-123 + | bd human dismiss bd-123 "No longer applicable" + | bd human dismiss bd-123 --reason "No longer applicable" + flag --reason type=string default="" usage="Reason for dismissal (optional)" + +bd human list + use: "list" + short: "List human-needed beads" + hidden: false + runnable: true + long: + | List issues labeled with 'human' tag. + | + | These are issues that require human intervention or input. Every + | human-labeled bead shows regardless of type (including gates and wisps). + | By default closed, pinned, and other done/frozen beads are hidden; use + | --status to select specific statuses, or --status=all to include every + | status. + | + | Examples: + | bd human list + | bd human list --status=closed + | bd human list --status=all + | bd human list --json + flag --status -s type=string default="" usage="Filter by status (open, closed, etc.; comma-separated for multiple, 'all' for every status)" + +bd human respond + use: "respond [response...]" + short: "Respond to a human-needed bead" + hidden: false + runnable: true + long: + | Respond to a human-needed bead by adding a comment and closing it. + | + | The response is added as a comment and the issue is closed with reason "Responded". + | The response text can be given as positional arguments, --response, --file, or --stdin. + | + | Examples: + | bd human respond bd-123 "Use OAuth2 for authentication" + | bd human respond bd-123 -r "Approved, proceed with implementation" + | bd human respond bd-123 --file response.md + | echo "Approved" | bd human respond bd-123 --stdin + flag --file type=string default="" annotations={cobra_annotation_mutually_exclusive=["stdin file response"]} usage="Read response text from file" + flag --response -r type=string default="" annotations={cobra_annotation_mutually_exclusive=["stdin file response"]} usage="Response text" + flag --stdin type=bool default="false" annotations={cobra_annotation_mutually_exclusive=["stdin file response"]} usage="Read response text from stdin" + +bd human stats + use: "stats" + short: "Show summary statistics for human-needed beads" + hidden: false + runnable: true + long: + | Display summary statistics for human-needed beads. + | + | Shows counts for total, pending (open), responded (closed without dismiss), + | and dismissed beads. + | + | Example: + | bd human stats + +bd import + use: "import [file|-]" + short: "Import issues from a JSONL file or stdin into the database" + group: "sync" + hidden: false + runnable: true + long: + | Import issues from a JSONL file (newline-delimited JSON) into the database. + | + | If no file is specified, imports from the configured import.path under .beads/ + | (default: issues.jsonl). Use "-" to read from stdin; redirecting stdin without + | "-" or a file argument is an error, so a typo'd 'bd import < file' cannot + | silently import the default file instead. This is the incremental counterpart to + | 'bd export': new issues are created and existing issues are updated (upsert + | semantics). + | + | Memory records (lines with "_type":"memory") are automatically detected and + | imported as persistent memories (equivalent to 'bd remember'). This makes + | 'bd export | bd import' a full round-trip for both issues and memories. + | + | Each JSONL line should map to an issue. The importer accepts every field + | 'bd export' emits — see 'bd export' output for the canonical schema. Only + | "title" is required; everything else is optional. + | + | Common fields: + | title Required. Short summary. + | description Long-form body. + | design, notes, Additional content sections. + | acceptance_criteria + | issue_type bug | feature | task | epic | chore | ... + | priority 0-4 (0 = critical). 0 is preserved (no omitempty). + | status open | in_progress | blocked | closed | ... + | (rows with status "tombstone" are skipped) + | assignee, owner, Ownership metadata. + | created_by + | labels Array of strings. + | dependencies Array of {issue_id, depends_on_id, type, ...}. + | comments Array of comment objects. + | external_ref, Cross-system identifiers (e.g. "gh-9"). + | source_system + | due_at, defer_until RFC3339 timestamps for scheduling. + | metadata Arbitrary JSON object preserved verbatim. + | + | Timestamps (created_at, updated_at, started_at, closed_at) are preserved + | when present in the JSONL and otherwise filled in by the importer. The + | legacy "wisp" boolean is accepted as an alias for "ephemeral". + | + | By default a row only rewrites an existing local issue when its + | updated_at is strictly newer. Older rows are skipped (reported as + | stale_skipped_ids) and rows with the same updated_at keep every local + | column — updated_at has second granularity, so a timestamp tie can be + | two distinct same-second updates, and the local row wins the tie + | (reported as tie_kept_local_ids; the row's labels/comments/dependencies + | still merge). The guard is also enforced inside the upsert itself, so a + | local update that lands while the import is running is preserved rather + | than overwritten. Existing issues that the import did rewrite are listed + | with a field-level summary (updated_issues), so local state changed by + | an import is visible. To deliberately restore an older snapshot, pass + | --allow-stale, which imports every row even when it overwrites newer + | local state. + | + | The staleness guard is import's ONLY overwrite guard: the per-field + | fences that bd update enforces (notes overwrite, live-claim reassign, + | close policy) do not apply to imported rows. Import replaces rows + | wholesale by design — a restore must not fail field-by-field — so + | --allow-stale is the single deliberate-overwrite opt-in, and + | updated_issues is where to look for what an import rewrote. + | + | Large imports are written in bounded transactions (a few hundred issues + | each, with a short pause between commits) with progress on stderr, so + | concurrent bd commands keep working while the import runs instead of + | stalling on one batch-wide write lock. Rows land in dependency order + | with their blocking edges in the same transaction, so a half-finished + | import never shows a blocked issue as ready. If an import fails partway, + | the already-committed chunks are durable and the command exits nonzero; + | re-running the same import is safe and converges (rows upsert, + | labels/comments/dependencies deduplicate). + | + | EXAMPLES: + | bd import # Import from configured import.path + | bd import backup.jsonl # Import from a specific file + | bd import -i backup.jsonl # Legacy alias for a specific file + | bd import - # Read JSONL from stdin + | cat issues.jsonl | bd import - # Pipe JSONL from another tool + | bd import --dry-run # Show what would be imported + | bd import --dedup # Skip issues with duplicate titles + | bd import --allow-stale old.jsonl # Restore an older snapshot (overwrites newer local rows) + | bd import --json # Structured output with created and skipped IDs + flag --allow-stale type=bool default="false" usage="Import rows even when older than the local issue (required to restore an older snapshot)" + flag --dedup type=bool default="false" usage="Skip lines whose title matches an existing open issue" + flag --dry-run type=bool default="false" usage="Show what would be imported without importing" + flag --input -i type=string default="" usage="Read JSONL from a specific file" + +bd info + use: "info" + short: "Show database information" + group: "setup" + hidden: false + runnable: true + long: + | Display information about the current database. + | + | This command helps debug issues where bd is using an unexpected database. It shows: + | - The absolute path to the database file + | - Database statistics (issue count) + | - Schema information (with --schema flag) + | - What's new in recent versions (with --whats-new flag) + | + | Examples: + | bd info + | bd info --json + | bd info --schema --json + | bd info --whats-new + | bd info --whats-new --json + | bd info --thanks + flag --schema type=bool default="false" usage="Include schema information in output" + flag --thanks type=bool default="false" usage="Show thank you page for contributors" + flag --whats-new type=bool default="false" usage="Show agent-relevant changes from recent versions" + +bd init + use: "init" + short: "Initialize bd in the current directory" + group: "setup" + hidden: false + runnable: true + long: + | Initialize bd in the current directory by creating a .beads/ directory + | and its storage (a Dolt database by default). Optionally specify a custom issue prefix. + | + | Dolt is the default and only supported storage backend, with full version + | control (history, branching, sync). + | + | Use --database to specify an existing server database name, overriding the + | default prefix-based naming. This is useful when an external tool (e.g. an orchestrator) + | has already created the database. + | + | With --stealth: configures per-repository git settings for invisible beads usage: + | • .git/info/exclude to prevent beads files from being committed + | Perfect for personal use without affecting repo collaborators. + | To set up a specific AI tool, run: bd setup --stealth + | + | By default, beads uses an embedded Dolt engine (no external server needed). + | Pass --server to use an external dolt sql-server instead. In server mode, + | set connection details with --server-host, --server-port, and --server-user. + | Password should be set via BEADS_DOLT_PASSWORD environment variable. + | + | Auto-export is optional. When enabled, bd exports issues to + | .beads/issues.jsonl after write commands (throttled to once per 60s). This is + | for viewers (bv), interchange, and issue-level migration; not backup. + | Cross-machine sync and backups use Dolt remotes/backups, not JSONL import/export. + | To enable: bd config set export.auto true + | + | Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): + | Skips all interactive prompts, using sensible defaults: + | • Role defaults to "maintainer" (override with --role) + | • Fork exclude auto-configured when fork detected + | • Auto-export left at default (disabled) + | • --contributor and --team flags are rejected (wizards require interaction) + | Also auto-detected when stdin is not a terminal or CI=true is set. + flag --agents-file type=string default="" usage="Custom filename for agent instructions (default: AGENTS.md)" + flag --agents-profile type=string default="" usage="AGENTS.md profile: 'minimal' (default, pointer to bd prime) or 'full' (complete command reference)" + flag --agents-template type=string default="" usage="Path to custom AGENTS.md template (overrides embedded default)" + flag --backend type=string default="" usage="Storage backend: dolt (default). Removed backends (postgres, mysql, sqlite) print migration guidance." + flag --contributor type=bool default="false" usage="Run OSS contributor setup wizard" + flag --debug type=bool default="false" usage="Run the managed Dolt sql-server with --loglevel=debug and CPU profiling (--prof cpu). Persisted to config.yaml as dolt.debug. No effect on externally-managed servers." + flag --destroy-token type=string default="" usage="Explicit confirmation token for destructive re-init in non-interactive mode (format: 'DESTROY-')" + flag --discard-remote type=bool default="false" usage="Authorize discarding the configured remote's Dolt history when re-initializing. Requires --destroy-token in non-interactive mode; see 'bd help init-safety'." + flag --external type=bool default="false" usage="Server is externally managed (skip server startup); use with --shared-server or --server" + flag --force type=bool default="false" usage="Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence or recreating a missing server-mode database (see 'bd help init-safety')." + flag --from-jsonl type=bool default="false" usage="Import issues from configured import.path; refuses remote history unless --discard-remote authorizes replacement" + flag --init-if-missing type=bool default="false" usage="If the workspace is already initialized, skip init and exit 0 instead of failing (idempotent init for scaffolds)" + flag --mysql-database type=string default="" hidden usage="Legacy MySQL database (removed backend compatibility only)" + flag --mysql-url type=string default="" hidden usage="Legacy MySQL connection URL (removed backend compatibility only)" + flag --non-interactive type=bool default="false" usage="Skip all interactive prompts (auto-detected in CI or non-TTY environments)" + flag --pg-schema type=string default="" hidden usage="Legacy PostgreSQL schema (removed backend compatibility only)" + flag --pg-url type=string default="" hidden usage="Legacy PostgreSQL connection URL (removed backend compatibility only)" + flag --prefix -p type=string default="" usage="Issue prefix (default: current directory name)" + flag --proxied-server type=bool default="false" usage="[EXPERIMENTAL] Use a per-workspace proxied dolt sql-server (proxy + child dolt) rooted at .beads/dolt" + flag --proxied-server-config-path type=string default="" usage="[EXPERIMENTAL] Absolute path to an existing dolt sql-server YAML config (proxied-server mode only). When set, bd uses this file instead of auto-generating one. Relative paths are rejected. Managed mode requires listener.host to be a numeric loopback IP (hostnames including localhost, non-loopback addresses, listener.socket, remotesapi, and cluster config are rejected); the same policy applies to BEADS_PROXIED_SERVER_CONFIG." + flag --proxied-server-external-host type=string default="" usage="[EXPERIMENTAL] Hostname or IP of an externally-managed dolt sql-server the proxy should front (proxied-server mode only). Mutually exclusive with --proxied-server-external-socket-path." + flag --proxied-server-external-keep-alive type=duration default="0s" usage="[EXPERIMENTAL] TCP keepalive period for the proxy→external connection. Zero uses the package default (30s)." + flag --proxied-server-external-port type=int default="0" usage="[EXPERIMENTAL] TCP port of the externally-managed dolt sql-server (proxied-server mode only). Required when --proxied-server-external-host is set." + flag --proxied-server-external-socket-path type=string default="" usage="[EXPERIMENTAL] Absolute unix socket path of the externally-managed dolt sql-server (proxied-server mode only). Mutually exclusive with --proxied-server-external-host. Relative paths are rejected." + flag --proxied-server-external-tls type=bool default="false" usage="[EXPERIMENTAL] Require TLS when connecting to the externally-managed dolt sql-server (proxied-server mode only)." + flag --proxied-server-external-tls-ca-cert-path type=string default="" usage="[EXPERIMENTAL] Absolute path to a CA certificate (PEM) used to verify the externally-managed dolt sql-server. Empty uses the system trust store. Relative paths are rejected." + flag --proxied-server-external-tls-cert-path type=string default="" usage="[EXPERIMENTAL] Absolute path to a client TLS certificate (for mTLS to the externally-managed dolt sql-server). Must be paired with --proxied-server-external-tls-key-path. Relative paths are rejected." + flag --proxied-server-external-tls-key-path type=string default="" usage="[EXPERIMENTAL] Absolute path to the client TLS private key (for mTLS to the externally-managed dolt sql-server). Must be paired with --proxied-server-external-tls-cert-path. Relative paths are rejected." + flag --proxied-server-external-tls-server-name type=string default="" usage="[EXPERIMENTAL] Server name to verify in the external dolt sql-server's TLS certificate. Defaults to the external host. Required with a unix socket unless --proxied-server-external-tls-skip-verify is set." + flag --proxied-server-external-tls-skip-verify type=bool default="false" usage="[EXPERIMENTAL] Skip TLS certificate verification for the external dolt sql-server. Insecure; testing only." + flag --proxied-server-external-user type=string default="" usage="[EXPERIMENTAL] MySQL user for the externally-managed dolt sql-server (proxied-server mode only). Defaults to \"root\" when empty. Password is read at runtime from $BEADS_PROXIED_SERVER_EXTERNAL_PASSWORD and is never persisted to disk." + flag --proxied-server-idle-timeout type=duration default="0s" usage="[EXPERIMENTAL] Idle duration after which the proxy shuts down its loopback listener and backend (proxied-server mode only). Omit for the built-in default (30s); 0 keeps the proxy and backend alive indefinitely; a positive value sets the window." + flag --proxied-server-log-path type=string default="" usage="[EXPERIMENTAL] Absolute path to the proxied dolt sql-server log file (proxied-server mode only). Default: /dolt/server.log. Relative paths are rejected." + flag --proxied-server-port type=int default="0" usage="[EXPERIMENTAL] Fixed TCP port for the proxy's loopback listener (proxied-server mode only). Default 0 = an OS-assigned free port. Startup fails if the port is already in use." + flag --proxied-server-root-path type=string default="" usage="[EXPERIMENTAL] Absolute directory holding the proxied dolt sql-server's lockfiles, pidfiles, and child .dolt repository (proxied-server mode only). Default: /dolt. May not exist yet — bd will create it. Relative paths are rejected." + flag --quiet -q type=bool default="false" usage="Suppress output (quiet mode)" + flag --recreate-missing type=bool default="false" usage="Explicitly authorize creating a fresh, empty database when this project's configured server-mode database is missing or unreachable. Opt-in per invocation only; never implied by --force, config, or env (see 'bd help init-safety')." + flag --reinit-local type=bool default="false" usage="Re-initialize local .beads/ over existing local data. Does NOT authorize remote divergence (see --discard-remote) or recreating a missing server-mode database (see --recreate-missing)." + flag --remote type=string default="" usage="Dolt remote URL to clone from and persist as sync.remote" + flag --role type=string default="" usage="Set beads role without prompting: \"maintainer\" or \"contributor\"" + flag --server type=bool default="false" usage="Use external dolt sql-server instead of embedded engine" + flag --server-host type=string default="" usage="Dolt server host (default: 127.0.0.1)" + flag --server-port type=int default="0" usage="Dolt server port (default: 3307)" + flag --server-socket type=string default="" usage="Unix domain socket path (overrides host/port; pass '' to ignore an ambient BEADS_DOLT_SERVER_SOCKET and use TCP)" + flag --server-tls type=bool default="false" usage="Require TLS for the init-time Dolt server connection (overrides BEADS_DOLT_SERVER_TLS for this run; not persisted - set the env var or credentials file for later commands)" + flag --server-user type=string default="" usage="Dolt server MySQL user (default: root)" + flag --setup-exclude type=bool default="false" usage="Configure .git/info/exclude to keep beads files local (for forks)" + flag --shared-server type=bool default="false" usage="Enable shared Dolt server mode (all projects share one server at ~/.beads/shared-server/)" + flag --skip-agents type=bool default="false" usage="Skip AGENTS.md and Claude/Codex/Cursor setup generation" + flag --skip-hooks type=bool default="false" usage="Skip git hooks installation" + flag --sqlite-path type=string default="" hidden usage="Legacy SQLite database file (removed backend compatibility only)" + flag --stealth type=bool default="false" usage="Enable stealth mode: keep beads files out of git via .git/info/exclude (sets no-git-ops)" + flag --team type=bool default="false" usage="Run team workflow setup wizard" + flag --team-server type=bool default="false" usage="[EXPERIMENTAL] The shared database's schema is managed by beads-team-server (bts): bd never creates the database or runs schema migrations, only verifies the schema version (proxied-server mode only). Not related to --team." + +bd init-safety + use: "init-safety" + short: "Explain bd init flag semantics and the destroy-token format" + hidden: false + runnable: true + long: + | bd init flag safety contract. + | + | Every bd init invocation resolves project_id from exactly one explicitly + | named source (local reinit, remote adoption, or a fresh mint). When the + | source is ambiguous, bd init refuses. + | + | FLAG SURFACE + | + | bd init Mint a new identity. Bootstraps from + | origin if it has refs/dolt/data. + | + | bd init --reinit-local Re-initialize local .beads/ over existing + | local data. Does NOT authorize discarding + | remote history. If origin has Dolt data + | this will refuse — pair with + | --discard-remote to override. Nor does it + | authorize recreating a missing server-mode + | database; that takes --recreate-missing. + | + | bd init --reinit-local \ Discard the remote's Dolt history and + | --discard-remote replace it with the local reinit. First + | bd dolt push after this will be a + | history-replacing force-push. + | + | bd init --force Deprecated alias for --reinit-local, with + | the same limits. Kept working for ≥2 + | releases. + | + | bd init --recreate-missing Authorize creating a fresh, empty database + | when this project's configured server-mode + | database is missing or unreachable. Opt-in + | per invocation only: never implied by + | --force/--reinit-local, config, or env. + | Without it, bd init REFUSES rather than + | silently recreating a lost database as + | empty (be-5up5). First rule out a stopped + | server or a wrong port or data dir, and + | restore if you can; the playbook is + | init-missing-server-db in + | docs/recovery/init-safety.md. Only when + | nothing is recoverable: + | + | bd init --recreate-missing --prefix + | + | bd init --from-jsonl Import from configured import.path. If + | origin has Dolt data, this refuses unless + | --discard-remote authorizes replacing that + | remote history. + | + | ADOPTING A REMOTE + | + | If you want to use the remote's existing history, use: + | + | bd bootstrap + | + | bd init will automatically suggest this when a remote is detected. + | + | DESTROY-TOKEN (non-interactive only) + | + | When running with no TTY (CI, agents, piped input), a destructive + | re-init requires an explicit --destroy-token value. That covers both + | --discard-remote and --reinit-local over existing issues. The token + | format is: + | + | DESTROY- + | + | For example, if your issue prefix is "bd", the token is "DESTROY-bd": + | + | bd init --reinit-local --discard-remote --destroy-token=DESTROY-bd + | + | In interactive (TTY) mode you confirm via a typed prompt instead. The + | token is not echoed by bd's runtime error messages — this is a + | deliberate guard against pattern-matched one-liners (see + | engdocs/adr/0002-init-safety-invariants.md). + | + | EXIT CODES + | + | 10 refused: remote has Dolt history and you selected local history + | without --discard-remote + | 11 refused: existing local data and you declined the destroy confirm + | (interactive mode only) + | 12 refused: destructive re-init (--discard-remote, or --reinit-local + | over existing issues) without a valid --destroy-token + | (non-interactive mode); also returned when the interactive + | --discard-remote typed-token confirmation is declined + | 15 refused: run directly in your home directory, which is not itself + | a git repository, with no BEADS_DIR named by the caller + | + | RECOVERY + | + | If you hit a refusal, see docs/recovery/init-safety.md for step-by-step recovery + | playbooks for each exit code. + | + | RE-CLONE GOTCHAS + | + | Setting a damaged or superseded database directory aside by hand, or + | relying on a fresh clone right away? Two gotchas from live recovery: + | + | Crash-loop: a set-aside store left INSIDE data_dir makes the sql-server + | treat it as a database and crash-loop with "root hash doesn't exist: + | ". Move it OUTSIDE data_dir instead. + | + | Missing tables: a fresh clone lacks clone-local tables (leases, wisps, + | events, ...) until you run "bd migrate schema" (no --force). You'll see + | "table not found: leases" until then; "Schema already at v" after + | running it is expected, not an error. + | + | See docs/recovery/init-safety.md#re-clone-gotchas for full detail. + | + +bd jira + use: "jira" + short: "Jira integration commands" + group: "advanced" + hidden: false + runnable: false + long: + | Synchronize issues between beads and Jira. + | + | Configuration: + | bd config set jira.url "https://company.atlassian.net" + | bd config set jira.project "PROJ" + | bd config set jira.projects "PROJ1,PROJ2" # Multiple projects + | bd config set jira.api_token "YOUR_TOKEN" + | bd config set jira.username "your_email@company.com" # For Jira Cloud + | bd config set jira.push_prefix "hippo" # Only push hippo-* issues to Jira + | bd config set jira.push_prefix "proj1,proj2" # Multiple prefixes (comma-separated) + | + | Environment variables (alternative to config): + | JIRA_API_TOKEN - Jira API token + | JIRA_USERNAME - Jira username/email + | JIRA_PROJECTS - Comma-separated project keys + | + | Examples: + | bd jira sync --pull # Import issues from Jira + | bd jira sync --push # Export issues to Jira + | bd jira sync # Bidirectional sync (pull then push) + | bd jira sync --dry-run # Preview sync without changes + | bd jira status # Show sync status + +bd jira pull + use: "pull [refs...]" + short: "Pull specific items from Jira" + hidden: false + runnable: true + long: + | Pull one or more items from Jira. + | + | Accepts bead IDs or external references as positional arguments. + | Equivalent to: bd jira sync --pull --issues + flag --dry-run type=bool default="false" usage="Preview pull without making changes" + +bd jira push + use: "push [bead-ids...]" + short: "Push specific beads to Jira" + hidden: false + runnable: true + long: + | Push one or more beads issues to Jira. + | + | Accepts bead IDs as positional arguments. + | Equivalent to: bd jira sync --push --issues + flag --dry-run type=bool default="false" usage="Preview push without making changes" + +bd jira status + use: "status" + short: "Show Jira sync status" + hidden: false + runnable: true + long: + | Show the current Jira sync status, including: + | - Last sync timestamp + | - Configuration status + | - Number of issues with Jira links + | - Issues pending push (no external_ref) + +bd jira sync + use: "sync" + short: "Synchronize issues with Jira" + hidden: false + runnable: true + long: + | Synchronize issues between beads and Jira. + | + | Modes: + | --pull Import issues from Jira into beads + | --push Export issues from beads to Jira + | (no flags) Bidirectional sync: pull then push, with conflict resolution + | + | Conflict Resolution: + | By default, newer timestamp wins. Override with: + | --prefer-local Always prefer local beads version + | --prefer-jira Always prefer Jira version + | + | Examples: + | bd jira sync --pull # Import from Jira + | bd jira sync --push --create-only # Push new issues only + | bd jira sync --dry-run # Preview without changes + | bd jira sync --prefer-local # Bidirectional, local wins + flag --create-only type=bool default="false" usage="Only create new issues, don't update existing" + flag --dry-run type=bool default="false" usage="Preview sync without making changes" + flag --issues type=string default="" usage="Comma-separated bead IDs to sync selectively (e.g., bd-abc,bd-def). Mutually exclusive with --parent." + flag --parent type=string default="" usage="Limit push to this bead and its descendants (push only). Mutually exclusive with --issues." + flag --prefer-jira type=bool default="false" usage="Prefer Jira version on conflicts" + flag --prefer-local type=bool default="false" usage="Prefer local version on conflicts" + flag --project type=stringSlice default="[]" usage="Project key(s) to sync (overrides configured project/projects)" + flag --pull type=bool default="false" usage="Pull issues from Jira" + flag --push type=bool default="false" usage="Push issues to Jira" + flag --state type=string default="all" usage="Issue state to sync: open, closed, all" + +bd kv + use: "kv" + short: "Key-value store commands" + group: "setup" + hidden: false + runnable: false + long: + | Commands for working with the beads key-value store. + | + | The key-value store is useful for storing flags, environment variables, + | or other user-defined data that persists across sessions. + | + | Examples: + | bd kv set mykey myvalue # Set a value + | bd kv get mykey # Get a value + | bd kv clear mykey # Delete a key + | bd kv list # List all key-value pairs + +bd kv clear + use: "clear " + short: "Delete a key-value pair" + hidden: false + runnable: true + long: + | Delete a key from the beads key-value store. + | + | Examples: + | bd kv clear feature_flag + | bd kv clear api_endpoint + +bd kv get + use: "get " + short: "Get a value by key" + hidden: false + runnable: true + long: + | Get a value from the beads key-value store. + | + | Examples: + | bd kv get feature_flag + | bd kv get api_endpoint + +bd kv list + use: "list" + short: "List all key-value pairs" + hidden: false + runnable: true + long: + | List all key-value pairs in the beads key-value store. + | + | With --prefix, list only the keys starting with that prefix. The filter is + | pushed into SQL on stores that support it, so a scoped read of a large kv + | store does not serialize the whole table. + | + | Examples: + | bd kv list + | bd kv list --json + | bd kv list --prefix mail.dog. --json + flag --prefix type=string default="" usage="list only keys starting with this prefix (filtered in SQL where supported)" + +bd kv set + use: "set " + short: "Set a key-value pair" + hidden: false + runnable: true + long: + | Set a key-value pair in the beads key-value store. + | + | This is useful for storing flags, environment variables, or other + | user-defined data that persists across sessions. + | + | Examples: + | bd kv set feature_flag true + | bd kv set api_endpoint https://api.example.com + | bd kv set max_retries 3 + +bd label + use: "label" + short: "Manage issue labels" + group: "issues" + hidden: false + runnable: false + +bd label add + use: "add [issue-id...] [label[,label...]]" + short: "Add one or more labels to one or more issues" + hidden: false + runnable: true + long: + | Add labels to issues. Issue IDs come first; the final argument is the label. Pass multiple labels comma-separated: bd label add bd-123 label1,label2 + +bd label list + use: "list [issue-id]" + short: "List labels for an issue" + hidden: false + runnable: true + +bd label list-all + use: "list-all" + short: "List all unique labels in the database" + hidden: false + runnable: true + +bd label propagate + use: "propagate [parent-id] [label]" + short: "Propagate a label from a parent issue to all its children" + hidden: false + runnable: true + long: + | Push a label from a parent down to all direct children that don't already have it. Useful for applying branch: labels across an epic's subtasks. + +bd label remove + use: "remove [issue-id...] [label[,label...]]" + short: "Remove one or more labels from one or more issues" + hidden: false + runnable: true + long: + | Remove labels from issues. Issue IDs come first; the final argument is the label. Pass multiple labels comma-separated: bd label remove bd-123 label1,label2 + | + | With --prefix, no label argument is needed: every label on the given issue(s) starting with the prefix is removed, e.g. bd label remove bd-123 --prefix pool:refused: + flag --prefix type=string default="" usage="Remove every label matching this prefix instead of an exact label (e.g., 'pool:refused:' removes all pool:refused:* labels). No label positional argument is needed when set. CAUTION: a short prefix removes every match on an issue in a single edit, including labels you may not have meant to touch (e.g. --prefix t removes both 'tier:opus' and 'test-needed') — check `bd label list ` first if unsure. State-dimension labels (e.g. 'launch:', 'scalegate:') should move via `bd set-state` instead, which this flag bypasses without its event-bead audit trail." + +bd label rename + use: "rename " + short: "Rename a label across every issue and wisp that carries it" + hidden: false + runnable: true + long: + | Rename a label everywhere it appears. An issue that already carries the new label keeps it and drops the old one instead of erroring - a merge, reported honestly (the write path measures merged from what the insert actually affected, never from a snapshot check, so its count is correct under concurrent writes). Pass --dry-run to preview the blast radius (a count and the first issues) without writing anything; the preview's merged count is a plain snapshot intersection of two separate reads, so treat it as an estimate, not the number the rename itself will report. + flag --dry-run type=bool default="false" usage="Preview the blast radius without renaming anything (merged count is a snapshot intersection, not authoritative - see --help)" + +bd linear + use: "linear" + short: "Linear integration commands" + group: "advanced" + hidden: false + runnable: false + long: + | Synchronize issues between beads and Linear. + | + | Configuration: + | bd config set linear.api_key "YOUR_API_KEY" + | bd config set linear.team_id "TEAM_ID" + | bd config set linear.team_ids "TEAM_ID1,TEAM_ID2" # Multiple teams (comma-separated) + | bd config set linear.project_id "PROJECT_ID" # Optional: sync only this project + | + | Environment variables (alternative to config): + | LINEAR_API_KEY - Linear API key (for individual developers) + | LINEAR_TEAM_ID - Linear team ID (UUID, singular) + | LINEAR_TEAM_IDS - Linear team IDs (comma-separated UUIDs) + | + | OAuth (for CI workers / automated sync): + | LINEAR_OAUTH_CLIENT_ID - OAuth app client ID + | LINEAR_OAUTH_CLIENT_SECRET - OAuth app client secret + | + | When both OAuth env vars are set, OAuth client_credentials flow is used + | instead of the API key. This allows CI workers to authenticate as an + | application (actor=application) rather than impersonating a user. + | Precedence: OAuth > LINEAR_API_KEY > config file. + | + | Data Mapping (optional, sensible defaults provided): + | Priority mapping (Linear 0-4 to Beads 0-4): + | bd config set linear.priority_map.0 4 # No priority -> Backlog + | bd config set linear.priority_map.1 0 # Urgent -> Critical + | bd config set linear.priority_map.2 1 # High -> High + | bd config set linear.priority_map.3 2 # Medium -> Medium + | bd config set linear.priority_map.4 3 # Low -> Low + | + | State mapping (Linear state type to Beads status): + | bd config set linear.state_map.backlog open + | bd config set linear.state_map.unstarted open + | bd config set linear.state_map.started in_progress + | bd config set linear.state_map.completed closed + | bd config set linear.state_map.canceled closed + | bd config set linear.state_map.my_custom_state in_progress # Custom state names + | + | Label to issue type mapping: + | bd config set linear.label_type_map.bug bug + | bd config set linear.label_type_map.feature feature + | bd config set linear.label_type_map.epic epic + | + | Relation type mapping (Linear relations to Beads dependencies): + | bd config set linear.relation_map.blocks blocks + | bd config set linear.relation_map.blockedBy blocks + | bd config set linear.relation_map.duplicate duplicates + | bd config set linear.relation_map.related related + | + | ID generation (optional, hash IDs to match bd/Jira hash mode): + | bd config set linear.id_mode "hash" # hash (default) + | bd config set linear.hash_length "6" # hash length 3-8 (default: 6) + | + | Examples: + | bd linear sync --pull # Import issues from Linear + | bd linear sync --push # Export issues to Linear + | bd linear sync # Bidirectional sync (pull then push) + | bd linear sync --dry-run # Preview sync without changes + | bd create "Fix login" --external-ref https://linear.app/team/issue/TEAM-123 + | # Link a local issue to an existing Linear issue + | bd linear status # Show sync status + +bd linear pull + use: "pull [refs...]" + short: "Pull specific items from Linear" + hidden: false + runnable: true + long: + | Pull one or more items from Linear. + | + | Accepts bead IDs or external references as positional arguments. + | Equivalent to: bd linear sync --pull --issues + flag --dry-run type=bool default="false" usage="Preview pull without making changes" + flag --relations type=bool default="false" usage="Import Linear relations as bd dependencies when pulling" + +bd linear push + use: "push [bead-ids...]" + short: "Push specific beads to Linear" + hidden: false + runnable: true + long: + | Push one or more beads issues to Linear. + | + | Accepts bead IDs as positional arguments. + | Equivalent to: bd linear sync --push --issues + flag --dry-run type=bool default="false" usage="Preview push without making changes" + +bd linear status + use: "status" + short: "Show Linear sync status" + hidden: false + runnable: true + long: + | Show the current Linear sync status, including: + | - Last sync timestamp + | - Configuration status + | - Number of issues with Linear links + | - Issues pending push (no external_ref) + +bd linear sync + use: "sync" + short: "Synchronize issues with Linear" + hidden: false + runnable: true + long: + | Synchronize issues between beads and Linear. + | + | Modes: + | --pull Import issues from Linear into beads + | --push Export issues from beads to Linear + | --pull-if-stale Pull only if data is stale (skip if fresh) + | (no flags) Bidirectional sync: pull then push, with conflict resolution + | + | Staleness (--pull-if-stale): + | --threshold 20m How old data must be before pulling (default 20m) + | A 5-minute debounce prevents agent loops: if a pull completed within 5 minutes, + | data is always treated as fresh regardless of the threshold. + | + | Team Selection: + | --team ID1,ID2 Override configured team IDs for this sync + | Multiple teams can be configured via linear.team_ids (comma-separated). + | Falls back to linear.team_id for backward compatibility. + | Push requires explicit --team when multiple teams are configured. + | + | Pull Options: + | --milestones Reconstruct Linear project milestones as local epic parents + | + | Type Filtering (--push only): + | --type task,feature Only sync issues of these types + | --exclude-type wisp Exclude issues of these types + | --include-ephemeral Include ephemeral issues (wisps, etc.); default is to exclude + | --parent TICKET Only push this ticket and its descendants + | --relations Import Linear relations as bd dependencies on pull + | + | Persistent push-direction ID filters (workflow artifacts, sandbox beads, etc.): + | bd config set linear.exclude_id_prefix "hw-mol-" + | bd config set linear.exclude_id_patterns "-wisp-,sandbox-,scratch-" + | + | exclude_id_prefix is a single case-sensitive prefix on the bead ID. + | exclude_id_patterns is a comma-separated list of case-sensitive substrings + | (matched anywhere in the ID). Both are combined as a union: a bead + | matching either rule is skipped from push (no create, no update). Beads + | with an existing external_ref that NOW match are silently skipped on + | future syncs; the Linear-side issue persists — archive/delete it manually + | if desired. + | + | Conflict Resolution: + | By default, newer timestamp wins. Override with: + | --prefer-local Always prefer local beads version + | --prefer-linear Always prefer Linear version + | + | Examples: + | bd linear sync --pull # Import from Linear + | bd linear sync --pull-if-stale # Pull only if data is stale + | bd linear sync --pull-if-stale --threshold 5m # Pull if older than 5 minutes + | bd linear sync --pull --relations # Import Linear blocking relations as bd deps + | bd linear sync --push --create-only # Push new issues only + | bd linear sync --push --type=task,feature # Push only tasks and features + | bd linear sync --push --exclude-type=wisp # Push all except wisps + | bd linear sync --push --parent=bd-abc123 # Push one ticket tree + | bd linear sync --dry-run # Preview without changes + | bd linear sync --prefer-local # Bidirectional, local wins + flag --create-only type=bool default="false" usage="Only create new issues, don't update existing" + flag --dry-run type=bool default="false" usage="Preview sync without making changes" + flag --exclude-type type=stringSlice default="[]" usage="Exclude issues of these types (can be repeated)" + flag --include-ephemeral type=bool default="false" usage="Include ephemeral issues (wisps, etc.) when pushing to Linear" + flag --issues type=string default="" usage="Comma-separated bead IDs to sync selectively (e.g., bd-abc,bd-def). Mutually exclusive with --parent." + flag --milestones type=bool default="false" usage="Reconstruct Linear project milestones as local epic parents when pulling" + flag --no-wait type=bool default="false" usage="Fail immediately if another sync is running instead of waiting" + flag --parent type=string default="" usage="Limit push to this beads ticket and its descendants" + flag --prefer-linear type=bool default="false" usage="Prefer Linear version on conflicts" + flag --prefer-local type=bool default="false" usage="Prefer local version on conflicts" + flag --pull type=bool default="false" usage="Pull issues from Linear" + flag --pull-if-stale type=bool default="false" usage="Pull only if Linear data is stale (skip if fresh)" + flag --push type=bool default="false" usage="Push issues to Linear" + flag --relations type=bool default="false" usage="Import Linear relations as bd dependencies when pulling" + flag --state type=string default="all" usage="Issue state to sync: open, closed, all" + flag --team type=stringSlice default="[]" usage="Team ID(s) to sync (overrides configured team_id/team_ids)" + flag --threshold type=duration default="20m0s" usage="Staleness threshold for --pull-if-stale (default 20m)" + flag --type type=stringSlice default="[]" usage="Only sync issues of these types (can be repeated)" + flag --update-refs type=bool default="true" usage="Update external_ref after creating Linear issues" + +bd linear teams + use: "teams" + short: "List available Linear teams" + hidden: false + runnable: true + long: + | List all teams accessible with your Linear API key. + | + | Use this to find the team ID (UUID) needed for configuration. + | + | Example: + | bd linear teams + | bd config set linear.team_id "12345678-1234-1234-1234-123456789abc" + +bd link + use: "link " + short: "Link two issues with a dependency" + group: "issues" + hidden: false + runnable: true + long: + | Link two issues with a dependency. + | + | Shorthand for 'bd dep add '. By default creates a "blocks" + | dependency (id2 blocks id1). Use --type to specify a different relationship. + | + | Examples: + | bd link bd-123 bd-456 # bd-456 blocks bd-123 + | bd link bd-123 bd-456 --type related # bd-123 related to bd-456 + | bd link bd-123 bd-456 --type parent-child + flag --type -t type=string default="blocks" usage="Dependency type (blocks|tracks|related|parent-child|discovered-from)" + +bd lint + use: "lint [issue-id...]" + short: "Check issues for missing template sections" + group: "views" + hidden: false + runnable: true + long: + | Check issues for missing recommended sections based on issue type. + | + | By default, lints all open issues. Specify issue IDs to lint specific issues. + | + | Section requirements by type: + | bug: Steps to Reproduce, Acceptance Criteria + | task: Acceptance Criteria + | feature: Acceptance Criteria + | epic: Success Criteria (or Acceptance Criteria) + | chore: (none) + | + | Additional per-type sections can be required via config; they are ADDITIVE + | to the built-ins above (built-in requirements are never relaxed): + | + | bd config set lint.sections.epic "Standards scorecard, Cost" + | + | Examples: + | bd lint # Lint all open issues + | bd lint bd-abc # Lint specific issue + | bd lint bd-abc bd-def # Lint multiple issues + | bd lint --type bug # Lint only bugs + | bd lint --status all # Lint all issues (including closed) + | + flag --status -s type=string default="" usage="Filter by status (default: open, use 'all' for all)" + flag --type -t type=string default="" usage="Filter by issue type (bug, task, feature, epic, decision, spike, story, chore, milestone)" + +bd list + use: "list" + short: "List issues" + group: "issues" + hidden: false + runnable: true + flag --all type=bool default="false" usage="Show all issues including closed (overrides default filter)" + flag --assignee -a type=string default="" usage="Filter by assignee" + flag --brief type=bool default="false" usage="Omit the free-form text (description, design, acceptance criteria, notes, payload, waiters) from each row. Filters that read those fields, such as --desc-contains, still select on them. An omitted field is indistinguishable from an empty one in --json; fetch a whole issue with bd show." + flag --closed-after type=string default="" usage="Filter issues closed after date (YYYY-MM-DD or RFC3339)" + flag --closed-before type=string default="" usage="Filter issues closed before date (YYYY-MM-DD or RFC3339)" + flag --created-after type=string default="" usage="Filter issues created after date (YYYY-MM-DD or RFC3339)" + flag --created-before type=string default="" usage="Filter issues created before date (YYYY-MM-DD or RFC3339)" + flag --defer-after type=string default="" usage="Filter issues deferred after date (supports relative: +6h, tomorrow)" + flag --defer-before type=string default="" usage="Filter issues deferred before date (supports relative: +6h, tomorrow)" + flag --deferred type=bool default="false" usage="Show only issues with defer_until set" + flag --deps type=string default="" no-opt-default="scheduling" usage="Annotate tree with dependency edges and order siblings by them: 'scheduling' (bare --deps) or 'all'" + flag --desc-contains type=string default="" usage="Filter by description substring (case-insensitive)" + flag --due-after type=string default="" usage="Filter issues due after date (supports relative: +6h, tomorrow)" + flag --due-before type=string default="" usage="Filter issues due before date (supports relative: +6h, tomorrow)" + flag --empty-description type=bool default="false" usage="Filter issues with empty or missing description" + flag --exclude-label type=stringSlice default="[]" usage="Exclude issues that have ANY of these labels" + flag --exclude-type type=stringSlice default="[]" usage="Exclude issue types from results (comma-separated or repeatable, e.g., --exclude-type=convoy,epic)" + flag --external-contains type=string default="" usage="Filter by external ref substring (case-insensitive)" + flag --external-ref type=string default="" usage="Filter by exact external_ref value" + flag --filter-parent type=string default="" hidden usage="Alias for --parent" + flag --flat type=bool default="false" usage="Disable tree format and use legacy flat list output" + flag --format type=string default="" usage="Export the dependency graph of the listed issues: 'digraph' (edge list for golang.org/x/tools/cmd/digraph), 'dot' (Graphviz), or a Go template rendered once per edge over .IssueID, .DependsOnID, .Type, .Issue, .Dependency. 'json' is the exception and does not export the graph: it is equivalent to --json, which also wins when both flags are given. 'digraph' and templates print nothing when no listed issue depends on another; 'dot' still prints the nodes" + flag --has-metadata-key type=string default="" usage="Filter issues that have this metadata key set" + flag --id type=string default="" usage="Filter by specific issue IDs (comma-separated or repeated, e.g., bd-1,bd-5,bd-10)" + flag --include-comments type=bool default="false" usage="Populate each row's comments field with full comment bodies in JSON output (--json only; costs one query per row that has comments). Without it, a row with comments carries comments_omitted=true, so an absent comments field is never mistaken for having none. Text output is unaffected either way." + flag --include-ephemeral type=bool default="false" usage="Include ephemeral wisp-plane rows in output (normally hidden)" + flag --include-gates type=bool default="false" usage="Include gate issues in output (normally hidden)" + flag --include-infra type=bool default="false" usage="Include infrastructure beads (agent/role/message) in output" + flag --include-templates type=bool default="false" usage="Include template molecules in output" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL). Can combine with --label-any" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE). Can combine with --label" + flag --label-pattern type=string default="" usage="Filter by label glob pattern (e.g., 'tech-*' matches tech-debt, tech-legacy)" + flag --label-regex type=string default="" usage="Filter by label regex pattern (e.g., 'tech-(debt|legacy)')" + flag --limit -n type=int default="50" usage="Limit results (an explicit --limit always wins, 0 meaning unlimited; otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50)" + flag --long type=bool default="false" usage="Show detailed multi-line output for each issue" + flag --max-rows type=int default="0" usage="Hard upper bound on rows returned. Returns a non-zero exit (code 2) and an error to stderr if exceeded. 0 disables (the default). Overrides BEADS_MAX_ROWS for this invocation. Useful in CI/agent rigs that want a circuit breaker against pathological queries. Honored on both the direct and the --proxied-server route." + flag --metadata-field type=stringArray default="[]" usage="Filter by metadata field (key=value, repeatable)" + flag --mol-type type=string default="" usage="Filter by molecule type: swarm, patrol, or work" + flag --no-assignee type=bool default="false" usage="Filter issues with no assignee" + flag --no-labels type=bool default="false" usage="Filter issues with no labels" + flag --no-pager type=bool default="false" usage="Disable pager output" + flag --no-parent type=bool default="false" usage="Exclude child issues (show only top-level issues)" + flag --no-pinned type=bool default="false" usage="Exclude pinned issues" + flag --notes-contains type=string default="" usage="Filter by notes substring (case-insensitive)" + flag --offset type=int default="0" usage="Skip the first N matching results (0-based). Only supported under --proxied-server." + flag --overdue type=bool default="false" usage="Show only issues with due_at in the past (not closed)" + flag --parent type=string default="" usage="Filter by parent issue ID (shows children of specified issue)" + flag --pinned type=bool default="false" usage="Show only pinned issues" + flag --pretty type=bool default="false" usage="Display issues in a tree format with status/priority symbols" + flag --priority -p type=string default="" usage="Priority (0-4 or P0-P4, 0=highest)" + flag --priority-max type=string default="" usage="Filter by maximum priority (inclusive, 0-4 or P0-P4)" + flag --priority-min type=string default="" usage="Filter by minimum priority (inclusive, 0-4 or P0-P4)" + flag --ready type=bool default="false" usage="Show only ready issues (no active blockers, same semantics as bd ready)" + flag --reverse -r type=bool default="false" usage="Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first)" + flag --skip-labels type=bool default="false" usage="Skip label hydration. The labels field in output will be empty regardless of actual labels. Use only when the caller does not depend on label data. Cannot combine with --label, --label-any, --label-pattern, --label-regex, --exclude-label, or --no-labels." + flag --sort type=string default="" usage="Sort by field: priority, created, updated, closed, status, id, title, type, assignee" + flag --spec type=string default="" usage="Filter by spec_id prefix" + flag --state type=string default="" hidden usage="Alias for --status" + flag --status -s type=string default="" usage="Filter by stored status (open, in_progress, blocked, deferred, closed). Comma-separated or repeated for multiple: --status open,in_progress" + flag --title type=string default="" usage="Filter by title text (case-insensitive substring match)" + flag --title-contains type=string default="" usage="Filter by title substring (case-insensitive)" + flag --tree type=bool default="true" usage="Hierarchical tree format (default: true; use --flat to disable)" + flag --type -t type=string default="" usage="Filter by type (bug, feature, task, epic, chore, decision, merge-request, molecule, gate, convoy). Aliases: mr→merge-request, feat→feature, mol→molecule, dec/adr→decision" + flag --updated-after type=string default="" usage="Filter issues updated after date (YYYY-MM-DD or RFC3339)" + flag --updated-before type=string default="" usage="Filter issues updated before date (YYYY-MM-DD or RFC3339)" + flag --watch -w type=bool default="false" usage="Watch for changes and auto-update display (implies --pretty)" + flag --wisp-type type=string default="" usage="Filter by wisp type: heartbeat, ping, patrol, gc_report, recovery, error, escalation" + +bd mail + use: "mail [subcommand] [args...]" + short: "Delegate to mail provider (e.g., gt mail)" + hidden: false + runnable: true + disable-flag-parsing: true + long: + | Delegates mail operations to an external mail provider. + | + | Agents often type 'bd mail' when working with beads, but mail functionality + | is typically provided by the orchestrator. This command bridges that gap + | by delegating to the configured mail provider. + | + | Configuration (checked in order): + | 1. BEADS_MAIL_DELEGATE or BD_MAIL_DELEGATE environment variable + | 2. 'mail.delegate' config setting (bd config set mail.delegate "gt mail") + | + | Examples: + | # Configure delegation (one-time setup) + | export BEADS_MAIL_DELEGATE="gt mail" + | # or + | bd config set mail.delegate "gt mail" + | + | # Then use bd mail as if it were gt mail + | bd mail inbox # Lists inbox + | bd mail send mayor/ -s "Hi" # Sends mail + | bd mail read msg-123 # Reads a message + +bd memories + use: "memories [search]" + short: "List or search persistent memories" + group: "setup" + hidden: false + runnable: true + long: + | List all memories, or search by keyword. + | + | Examples: + | bd memories # list all memories + | bd memories dolt # search for memories about dolt + | bd memories "race flag" # search for a phrase + +bd merge-slot + use: "merge-slot" + short: "Manage merge-slot gates for serialized conflict resolution" + group: "issues" + hidden: false + runnable: false + long: + | Merge-slot gates serialize conflict resolution in the merge queue. + | + | A merge slot is an exclusive access primitive: only one agent can hold it at a time. + | This prevents "monkey knife fights" where multiple polecats race to resolve conflicts + | and create cascading conflicts. + | + | Each rig has one merge slot bead: -merge-slot (labeled gt:slot). + | The slot uses: + | - status=open: slot is available + | - status=in_progress: slot is held + | - metadata.holder: who currently holds the slot + | - metadata.waiters: priority-ordered queue of waiters + | + | Examples: + | bd merge-slot create # Create merge slot for current rig + | bd merge-slot check # Check if slot is available + | bd merge-slot acquire # Try to acquire the slot + | bd merge-slot release # Release the slot + +bd merge-slot acquire + use: "acquire" + short: "Acquire the merge slot" + hidden: false + runnable: true + long: + | Attempt to acquire the merge slot for exclusive access. + | + | If the slot is available (status=open), it will be acquired: + | - status set to in_progress + | - holder set to the requester + | + | If the slot is held (status=in_progress), the command fails unless + | --wait is passed, which adds the requester to the waiters queue. + | + | Use --holder to specify who is acquiring (default: BEADS_ACTOR env var). + flag --holder type=string default="" usage="Who is acquiring the slot (default: BEADS_ACTOR)" + flag --wait type=bool default="false" usage="Add to waiters list if slot is held" + +bd merge-slot check + use: "check" + short: "Check merge slot availability" + hidden: false + runnable: true + long: + | Check if the merge slot is available or held. + | + | Returns: + | - available: slot can be acquired + | - held by : slot is currently held + | - not found: no merge slot exists for this rig + +bd merge-slot create + use: "create" + short: "Create a merge slot bead for the current rig" + hidden: false + runnable: true + long: + | Create a merge slot bead for serialized conflict resolution. + | + | The slot ID is automatically generated based on the beads prefix (e.g., gt-merge-slot). + | The slot is created with status=open (available). + +bd merge-slot release + use: "release" + short: "Release the merge slot" + hidden: false + runnable: true + long: + | Release the merge slot after conflict resolution is complete. + | + | Sets status back to open and clears the holder field. + | If there are waiters, the highest-priority waiter should then acquire. + flag --holder type=string default="" usage="Who is releasing the slot (for verification)" + +bd metrics + use: "metrics" + short: "Show or change anonymous usage-metrics settings" + hidden: false + runnable: true + long: + | Show whether anonymous usage metrics are on, see exactly what is sent, and + | turn them on or off. + | + | bd shares anonymous usage metrics to learn how people actually use it — just + | which commands get run, plus the bd version and OS platform. That's how we decide + | what to polish next. We never collect your issues, paths, remotes, identity, or + | any user-supplied text. + | + | bd metrics show the current status and what is collected + | bd metrics on turn metrics on + | bd metrics off turn metrics off + | bd metrics example show real examples of the events bd sends + +bd metrics example + use: "example" + short: "Show real examples of the anonymous metrics bd sends" + hidden: false + runnable: true + +bd metrics off + use: "off" + short: "Turn anonymous usage metrics off" + hidden: false + runnable: true + +bd metrics on + use: "on" + short: "Turn anonymous usage metrics on" + hidden: false + runnable: true + +bd migrate + use: "migrate" + short: "Database migration commands" + group: "maint" + hidden: false + runnable: true + long: + | Database migration and data transformation commands. + | + | Without subcommand, checks and updates database metadata to current version. + | + | Subcommands: + | hooks Plan git hook migration to marker-managed format + | issues Move issues between repositories + | schema Apply pending schema migrations (idempotent) + | sync Set up sync.branch workflow for multi-clone setups + | from-server-to-proxied-server [EXPERIMENTAL] Switch server mode to proxied-server mode + | from-proxied-server-to-server [EXPERIMENTAL] Switch proxied-server mode to server mode + | from-shared-server-to-proxied-server [EXPERIMENTAL] Switch shared-server mode to proxied-server mode + | from-proxied-server-to-shared-server [EXPERIMENTAL] Switch proxied-server mode to shared-server mode + | + | On a remote-backed database with pending schema migrations bd refuses to + | migrate in place (#4259): migrating two clones independently forks the schema + | so bd dolt pull can no longer merge — the break is silent and unrecoverable. + | Use --force to confirm you are the single designated migrator, after which you + | should publish the migrated schema with 'bd dolt push'. The env-var equivalent + | BD_ALLOW_REMOTE_MIGRATE=1 remains supported for scripted/CI use. + | + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + flag --force type=bool default="false" usage="Bypass the remote-migrate gate as the single designated migrator (equivalent to BD_ALLOW_REMOTE_MIGRATE=1)" + flag --inspect type=bool default="false" usage="Show migration plan and database state for AI agent analysis" + flag --update-repo-id type=bool default="false" usage="Update repository ID (use after changing git remote)" + flag --yes type=bool default="false" usage="Auto-confirm prompts" + +bd migrate from-proxied-server-to-server + use: "from-proxied-server-to-server" + short: "[EXPERIMENTAL] Switch a proxied-server repo to server mode" + hidden: false + runnable: true + long: + | Switch a repo from proxied-server mode to server mode (bd init --server). + | + | Both modes root their dolt sql-server at the same .beads/dolt directory, so this + | only rewrites .beads/metadata.json (dolt_mode) and removes the proxied-server + | sidecar — no Dolt data is copied or moved. Stop the running proxy first with + | 'bd dolt stop'. + | + | Note: dolt_mode lives in the committed metadata.json, so this change propagates + | to clones on the next push. + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + +bd migrate from-proxied-server-to-shared-server + use: "from-proxied-server-to-shared-server" + short: "[EXPERIMENTAL] Switch a proxied-server repo back to shared-server mode" + hidden: false + runnable: true + long: + | Switch a repo from proxied-server mode back to shared-server mode. + | + | Only applies to a proxied-server repo rooted at the shared dolt directory + | (~/.beads/shared-server/dolt) — the reverse of from-shared-server-to-proxied-server. + | This rewrites .beads/metadata.json (dolt_mode), re-enables dolt.shared-server, and + | removes the proxied-server sidecar; no Dolt data is copied or moved. Stop the + | running proxy first with 'bd dolt stop'. + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + +bd migrate from-server-to-proxied-server + use: "from-server-to-proxied-server" + short: "[EXPERIMENTAL] Switch a server-mode repo to proxied-server mode" + hidden: false + runnable: true + long: + | Switch a repo from server mode (bd init --server) to proxied-server mode. + | + | Both modes root their dolt sql-server at the same .beads/dolt directory, so this + | only rewrites .beads/metadata.json (dolt_mode) and writes the proxied-server + | sidecar — no Dolt data is copied or moved. Stop the running server first with + | 'bd dolt stop'. + | + | Note: dolt_mode lives in the committed metadata.json, so this change propagates + | to clones on the next push. + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + flag --idle-timeout type=duration default="0s" usage="Proxy idle timeout; omit for the 30s default, 0 for indefinite uptime" + +bd migrate from-shared-server-to-proxied-server + use: "from-shared-server-to-proxied-server" + short: "[EXPERIMENTAL] Switch a shared-server repo to proxied-server mode" + hidden: false + runnable: true + long: + | Switch a repo from shared-server mode to proxied-server mode. + | + | The proxied server is rooted at the shared dolt directory + | (~/.beads/shared-server/dolt), so no Dolt data is copied or moved; this rewrites + | .beads/metadata.json (dolt_mode), turns off dolt.shared-server for this repo, and + | writes the proxied-server sidecar. Stop the running shared server first with + | 'bd dolt stop' — note that stops it for every project sharing it. + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + flag --idle-timeout type=duration default="0s" usage="Proxy idle timeout; omit for the 30s default, 0 for indefinite uptime" + +bd migrate hooks + use: "hooks [path]" + short: "Plan or apply git hook migration to marker-managed format" + hidden: false + runnable: true + long: + | Analyze git hook files and sidecar artifacts for migration to marker-managed format. + | + | Modes: + | --dry-run Preview migration operations without changing files + | --apply Apply migration operations + | + | Examples: + | bd migrate hooks --dry-run + | bd migrate hooks --apply + | bd migrate hooks --apply --yes + | bd migrate hooks --dry-run --json + flag --apply type=bool default="false" usage="Apply planned hook migration changes" + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + flag --yes type=bool default="false" usage="Skip confirmation prompt for --apply" + +bd migrate issues + use: "issues" + short: "Move issues between repositories" + hidden: false + runnable: true + long: + | Move issues from one source repository to another with filtering and dependency preservation. + | + | This command updates the source_repo field for selected issues, allowing you to: + | - Move contributor planning issues to upstream repository + | - Reorganize issues across multi-phase repositories + | - Consolidate issues from multiple repos + | + | Examples: + | # Preview migration from planning repo to current repo + | bd migrate-issues --from ~/.beads-planning --to . --dry-run + | + | # Move all open P1 bugs + | bd migrate-issues --from ~/repo1 --to ~/repo2 --priority 1 --type bug --status open + | + | # Move specific issues with their dependencies + | bd migrate-issues --from . --to ~/archive --id bd-abc --id bd-xyz --include closure + | + | # Move issues with label filter + | bd migrate-issues --from . --to ~/feature-work --label frontend --label urgent + flag --dry-run type=bool default="false" usage="Show plan without making changes" + flag --from type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Source repository (required)" + flag --id type=stringSlice default="[]" usage="Specific issue IDs to migrate (can specify multiple)" + flag --ids-file type=string default="" usage="File containing issue IDs (one per line)" + flag --include type=string default="none" usage="Include dependencies: none/upstream/downstream/closure" + flag --label type=stringSlice default="[]" usage="Filter by labels (can specify multiple)" + flag --priority type=int default="-1" usage="Filter by priority (0-4)" + flag --status type=string default="" usage="Filter by status (open/closed/all)" + flag --strict type=bool default="false" usage="Fail on orphaned dependencies or missing repos" + flag --to type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Destination repository (required)" + flag --type type=string default="" usage="Filter by issue type (bug/feature/task/epic/chore/decision)" + flag --within-from-only type=bool default="true" usage="Only include dependencies from source repo" + flag --yes type=bool default="false" usage="Skip confirmation prompt" + +bd migrate legacy-sqlite + use: "legacy-sqlite --source-db PATH --output PATH|-" + short: "Read an authenticated legacy SQLite database as JSONL" + hidden: false + runnable: true + annotations: {bd:skip_store="1"} + flag --output type=string default="" usage="JSONL output path, or - for stdout" + flag --source-db type=string default="" usage="Legacy SQLite database (read-only)" + +bd migrate schema + use: "schema" + short: "Apply pending schema migrations (idempotent)" + hidden: false + runnable: true + long: + | Apply pending schema migrations idempotently. + | + | bd does not apply pending schema migrations to an existing database without + | the operator's consent, and running this command is that consent + | (BD_ALLOW_MIGRATE=1 is the scripted form). On a remote-backed database the + | remote-migrate gate still applies: --force confirms you are the single + | designated migrator. Migrating is one-way: bd releases built for an older + | schema refuse the migrated database. + | + | On an up-to-date database this is a no-op, so it is safe to run in CI, + | release gates, and recovery scenarios. + | + | Example: + | bd migrate schema + | bd migrate schema --json + flag --force type=bool default="false" usage="Bypass the remote-migrate gate as the single designated migrator (equivalent to BD_ALLOW_REMOTE_MIGRATE=1)" + +bd migrate sync + use: "sync " + short: "Set up sync.branch workflow for multi-clone setups" + hidden: false + runnable: true + long: + | Configure separate branch workflow for multi-clone setups. + | + | This sets the sync.branch config value so that issue data is committed + | to a dedicated branch, keeping your main branch clean. + | + | Example: + | bd migrate sync beads-sync + flag --dry-run type=bool default="false" usage="Show what would be done without making changes" + +bd migrate-issues + use: "migrate-issues" + short: "Move issues between repositories" + hidden: true + deprecated: "use 'bd migrate issues' instead (will be removed in v1.0.0)" + runnable: true + long: + | Move issues from one source repository to another with filtering and dependency preservation. + | + | This command updates the source_repo field for selected issues, allowing you to: + | - Move contributor planning issues to upstream repository + | - Reorganize issues across multi-phase repositories + | - Consolidate issues from multiple repos + | + | Examples: + | # Preview migration from planning repo to current repo + | bd migrate-issues --from ~/.beads-planning --to . --dry-run + | + | # Move all open P1 bugs + | bd migrate-issues --from ~/repo1 --to ~/repo2 --priority 1 --type bug --status open + | + | # Move specific issues with their dependencies + | bd migrate-issues --from . --to ~/archive --id bd-abc --id bd-xyz --include closure + | + | # Move issues with label filter + | bd migrate-issues --from . --to ~/feature-work --label frontend --label urgent + flag --dry-run type=bool default="false" usage="Show plan without making changes" + flag --from type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Source repository (required)" + flag --id type=stringSlice default="[]" usage="Specific issue IDs to migrate (can specify multiple)" + flag --ids-file type=string default="" usage="File containing issue IDs (one per line)" + flag --include type=string default="none" usage="Include dependencies: none/upstream/downstream/closure" + flag --label type=stringSlice default="[]" usage="Filter by labels (can specify multiple)" + flag --priority type=int default="-1" usage="Filter by priority (0-4)" + flag --status type=string default="" usage="Filter by status (open/closed/all)" + flag --strict type=bool default="false" usage="Fail on orphaned dependencies or missing repos" + flag --to type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Destination repository (required)" + flag --type type=string default="" usage="Filter by issue type (bug/feature/task/epic/chore/decision)" + flag --within-from-only type=bool default="true" usage="Only include dependencies from source repo" + flag --yes type=bool default="false" usage="Skip confirmation prompt" + +bd migrate-personal + use: "migrate-personal" + short: "Move personal planning issues from the project database to your planning repo" + group: "setup" + hidden: false + runnable: true + long: + | Identify issues you created in the project database and move them to your + | personal planning repository (~/.beads-planning by default). + | + | This is a one-time migration for contributors who created personal planning + | issues before contributor routing was configured. + | + | The command: + | 1. Finds all issues in the project database created by your git identity + | 2. Shows you the list and asks for confirmation + | 3. Moves them to the planning repo configured in routing.contributor + | + | EXAMPLES: + | bd migrate-personal # Interactive: show list and prompt + | bd migrate-personal -y # Non-interactive: skip confirmation + flag --yes -y type=bool default="false" usage="Skip confirmation prompt" + +bd mol + use: "mol" + aliases: ["protomolecule"] + short: "Molecule commands (work templates)" + hidden: false + runnable: false + long: + | Manage molecules - work templates for agent workflows. + | + | Protos are template epics with the "template" label. They define a DAG of work + | that can be spawned to create real issues (molecules). + | + | The molecule metaphor: + | - A proto is an uninstantiated template (reusable work pattern) + | - Spawning creates a molecule (real issues) from the proto + | - Variables ({{key}}) are substituted during spawning + | - Bonding combines protos or molecules into compounds + | - Distilling extracts a proto from an ad-hoc epic + | + | Commands: + | show Show proto/molecule structure and variables + | pour Instantiate proto as persistent mol (liquid phase) + | wisp Instantiate proto as ephemeral wisp (vapor phase) + | bond Polymorphic combine: proto+proto, proto+mol, mol+mol + | squash Condense molecule to digest + | burn Discard wisp + | distill Extract proto from ad-hoc epic + | + | Use "bd formula list" to list available formulas. + +bd mol bond + use: "bond " + aliases: ["fart"] + short: "Bond two protos or molecules together" + hidden: false + runnable: true + long: + | Bond two protos or molecules to create a compound. + | + | The bond command is polymorphic - it handles different operand types: + | + | formula + formula → cook both, compound proto + | formula + proto → cook formula, compound proto + | formula + mol → cook formula, spawn and attach + | proto + proto → compound proto (reusable template) + | proto + mol → spawn proto, attach to molecule + | mol + proto → spawn proto, attach to molecule + | mol + mol → join into compound molecule + | + | Formula names (e.g., mol-polecat-arm) are cooked inline as ephemeral protos. + | This avoids needing pre-cooked proto beads in the database. + | + | Bond types: + | sequential (default) - B runs after A completes + | parallel - B runs alongside A + | conditional - B runs only if A fails + | + | With --ref, the spawned proto arm is nested inside the molecule operand + | rather than ordered against it - whichever side of the bond that is - so a + | sequential --ref arm carries no ordering: it is ready as soon as it is + | spawned. Ordering a nested arm against its own container is unsatisfiable + | in both directions (the arm waits for the molecule to close; the molecule + | cannot close while it holds an open arm). --ref is refused with --type + | conditional, which has no safe degradation - omit --ref to bond a + | conditional arm as a sibling. + | + | Phase control: + | By default, spawned protos follow the target's phase: + | - Attaching to mol (Ephemeral=false) → spawns as persistent (Ephemeral=false) + | - Attaching to ephemeral issue (Ephemeral=true) → spawns as ephemeral (Ephemeral=true) + | + | Override with: + | --pour Force spawn as liquid (persistent, Ephemeral=false) + | --ephemeral Force spawn as vapor (ephemeral, Ephemeral=true, excluded from Dolt sync via dolt_ignore) + | + | Dynamic bonding (Christmas Ornament pattern): + | Use --ref to specify a custom child reference with variable substitution. + | This creates IDs like "parent.child-ref" instead of random hashes. + | + | Example: + | bd mol bond mol-worker-arm bd-patrol --ref arm-{{worker_name}} --var worker_name=ace + | # Creates: bd-patrol.arm-ace (and children like bd-patrol.arm-ace.capture) + | + | Use cases: + | - Found important bug during patrol? Use --pour to persist it + | - Need ephemeral diagnostic on persistent feature? Use --ephemeral + | - Spawning per-worker arms on a patrol? Use --ref for readable IDs + | + | Examples: + | bd mol bond mol-feature mol-deploy # Compound proto + | bd mol bond mol-feature mol-deploy --type parallel # Run in parallel + | bd mol bond mol-feature bd-abc123 # Attach proto to molecule + | bd mol bond bd-abc123 bd-def456 # Join two molecules + | bd mol bond mol-critical-bug wisp-patrol --pour # Persist found bug + | bd mol bond mol-temp-check bd-feature --ephemeral # Ephemeral diagnostic + | bd mol bond mol-arm bd-patrol --ref arm-{{name}} --var name=ace # Dynamic child ID + flag --as type=string default="" usage="Custom title for compound proto (proto+proto only)" + flag --dry-run type=bool default="false" usage="Preview what would be created" + flag --ephemeral type=bool default="false" usage="Force spawn as vapor (ephemeral, Ephemeral=true)" + flag --pour type=bool default="false" usage="Force spawn as liquid (persistent, Ephemeral=false)" + flag --ref type=string default="" usage="Custom child reference with {{var}} substitution (e.g., arm-{{polecat_name}})" + flag --type type=string default="sequential" usage="Bond type: sequential, parallel, or conditional" + flag --var type=stringArray default="[]" usage="Variable substitution for spawned protos (key=value)" + +bd mol burn + use: "burn [molecule-id...]" + short: "Delete a molecule without creating a digest" + hidden: false + runnable: true + long: + | Burn a molecule, deleting it without creating a digest. + | + | Unlike squash (which creates a permanent digest before deletion), burn + | completely removes the molecule with no trace. Use this for: + | - Abandoned patrol cycles + | - Crashed or failed workflows + | - Test/debug molecules you don't want to preserve + | + | The burn operation differs based on molecule phase: + | - Wisp (ephemeral): Direct delete + | - Mol (persistent): Cascade delete (syncs to remotes) + | + | CAUTION: This is a destructive operation. The molecule's data will be + | permanently lost. If you want to preserve a summary, use 'bd mol squash'. + | + | Example: + | bd mol burn bd-abc123 # Delete molecule with no trace + | bd mol burn bd-abc123 --dry-run # Preview what would be deleted + | bd mol burn bd-abc123 --force # Skip confirmation + | bd mol burn bd-a1 bd-b2 bd-c3 # Batch delete multiple wisps + flag --dry-run type=bool default="false" usage="Preview what would be deleted" + flag --force type=bool default="false" usage="Skip confirmation prompt" + flag --yes -y type=bool default="false" hidden usage="Alias for --force (skip confirmation)" + +bd mol current + use: "current [molecule-id]" + short: "Show current position in molecule workflow" + hidden: false + runnable: true + long: + | Show where you are in a molecule workflow. + | + | If molecule-id is given, show status for that molecule. + | If not given, infer from in_progress issues assigned to current agent. + | + | The output shows all steps with status indicators: + | [done] - Step is complete (closed) + | [current] - Step is in_progress (you are here) + | [ready] - Step is ready to start (unblocked) + | [blocked] - Step is blocked by dependencies + | [pending] - Step is waiting + | + | For large molecules (>100 steps), a summary is shown instead. + | Use --limit or --range to view specific steps: + | bd mol current --limit 50 # Show first 50 steps + | bd mol current --range 100-150 # Show steps 100-150 + flag --for type=string default="" usage="Show molecules for a specific agent/assignee" + flag --limit type=int default="0" usage="Maximum number of steps to display (0 = auto, use 'all' threshold)" + flag --range type=string default="" usage="Display specific step range (e.g., '1-50', '100-150')" + +bd mol distill + use: "distill [formula-name]" + short: "Extract a formula from an existing epic" + hidden: false + runnable: true + long: + | Distill a molecule by extracting a reusable formula from an existing epic. + | + | This is the reverse of pour: instead of formula → molecule, it's molecule → formula. + | + | The distill command: + | 1. Loads the existing epic and all its children + | 2. Converts the structure to a .formula.json file + | 3. Replaces concrete values with {{variable}} placeholders (via --var flags) + | + | Use cases: + | - Team develops good workflow organically, wants to reuse it + | - Capture tribal knowledge as executable templates + | - Create starting point for similar future work + | + | Variable syntax (both work - we detect which side is the concrete value): + | --var branch=feature-auth Spawn-style: variable=value (recommended) + | --var feature-auth=branch Substitution-style: value=variable + | + | Output locations (first writable wins): + | 1. /formulas/ (project-level, default) + | 2. /.beads/formulas/ (repo-local formulas) + | 3. ~/.beads/formulas/ (user-level, if project not writable) + | + | Examples: + | bd mol distill bd-o5xe my-workflow + | bd mol distill bd-abc release-workflow --var feature_name=auth-refactor + flag --dry-run type=bool default="false" usage="Preview what would be created" + flag --output type=string default="" usage="Output directory for formula file" + flag --var type=stringArray default="[]" usage="Replace value with {{variable}} placeholder (variable=value)" + +bd mol last-activity + use: "last-activity " + short: "Show last activity timestamp for a molecule" + hidden: false + runnable: true + long: + | Show the most recent activity timestamp for a molecule. + | + | Returns the timestamp of the most recent change to any step in the molecule, + | making it easy to detect stale or stuck molecules. + | + | Activity sources: + | step_closed - A step was closed + | step_updated - A step was updated (claimed, edited, etc.) + | molecule_updated - The molecule root itself was updated + | + | Examples: + | bd mol last-activity hq-wisp-0laki + | bd mol last-activity hq-wisp-0laki --json + +bd mol pour + use: "pour " + short: "Instantiate a proto as a persistent mol (solid -> liquid)" + hidden: false + runnable: true + long: + | Pour a proto into a persistent mol - like pouring molten metal into a mold. + | + | This is the chemistry-inspired command for creating PERSISTENT work from templates. + | The resulting mol is stored as persistent beads in the issue database and + | syncs like any other bead (bd dolt push / pull). + | + | Phase transition: Proto (solid) -> pour -> Mol (liquid) + | + | WHEN TO USE POUR vs WISP: + | pour (liquid): Persistent work that needs audit trail + | - Feature implementations spanning multiple sessions + | - Work you may need to reference later + | - Anything worth preserving in git history + | + | wisp (vapor): Ephemeral work that auto-cleans up + | - Release workflows (one-time execution) + | - Operational loops and recurring cycles + | - Health checks and diagnostics + | - Any operational workflow without audit value + | + | TIP: Formulas can specify phase:"vapor" to recommend wisp usage. + | If you pour a vapor-phase formula, you'll get a warning. + | + | Examples: + | bd mol pour mol-feature --var name=auth # Persistent feature work + | bd mol pour mol-review --var pr=123 # Persistent code review + flag --assignee type=string default="" usage="Assign the root issue to this agent/user" + flag --attach type=stringSlice default="[]" usage="Proto to attach after spawning (repeatable)" + flag --attach-type type=string default="sequential" usage="Bond type for attachments: sequential, parallel, or conditional" + flag --dry-run type=bool default="false" usage="Preview what would be created" + flag --var type=stringArray default="[]" usage="Variable substitution (key=value); a name the proto cannot consume is an error" + +bd mol progress + use: "progress [molecule-id]" + short: "Show molecule progress summary" + hidden: false + runnable: true + long: + | Show efficient progress summary for a molecule. + | + | This command uses indexed queries to count progress without loading all steps, + | making it suitable for very large molecules (millions of steps). + | + | If no molecule-id is given, shows progress for any molecule you're working on. + | + | Output includes: + | - Progress: completed / total (percentage) + | - Current step: the in-progress step (if any) + | - Rate: steps/hour based on closure times + | - ETA: estimated time to completion + | + | Example: + | bd mol progress bd-hanoi-xyz + +bd mol ready + use: "ready --gated" + short: "Find molecules ready for gate-resume dispatch" + hidden: false + runnable: true + long: + | Find molecules where a gate has closed and the workflow is ready to resume. + | + | This command discovers molecules waiting at a gate step where: + | 1. The molecule has a gate bead that blocks a step + | 2. The gate bead is now closed (condition satisfied) + | 3. The blocked step is now ready to proceed + | 4. No agent currently has this molecule hooked + | + | This enables discovery-based resume without explicit waiter tracking. + | The patrol system uses this to find and dispatch gate-ready molecules. + | + | Examples: + | bd mol ready --gated # Find all gate-ready molecules + | bd mol ready --gated --json # JSON output for automation + flag --gated type=bool default="false" usage="Find molecules ready for gate-resume dispatch (always on for this subcommand)" + +bd mol seed + use: "seed " + short: "Verify formula accessibility" + hidden: false + runnable: true + long: + | Verify that a formula is accessible and can be cooked. + | + | The seed command checks formula search paths to ensure a formula exists + | and can be loaded. This is useful for verifying system health before + | attempting to spawn work from a formula. + | + | Formula search paths (checked in order): + | 1. /formulas/ (active project) + | 2. /.beads/formulas/ (repo-local formulas) + | 3. ~/.beads/formulas/ (user level) + | 4. $GT_ROOT/.beads/formulas/ (shared workspace root, if GT_ROOT set) + | + | Examples: + | bd mol seed mol-feature # Verify specific formula + | bd mol seed mol-review --var name=test # Verify with variable substitution + flag --var type=stringArray default="[]" usage="Variable substitution for condition filtering (key=value)" + +bd mol show + use: "show " + short: "Show molecule details" + hidden: false + runnable: true + long: + | Show molecule structure and details. + | + | A molecule's steps are the issues linked to its root by parent-child + | dependencies (bd create --parent, bd mol pour); an id that merely looks + | hierarchical, like bd-xyz.4, does not make an issue a step. + | + | The --parallel flag highlights parallelizable steps: + | - Steps with no blocking dependencies can run in parallel + | - Shows which steps are ready to start now + | - Identifies parallel groups (steps that can run concurrently) + | + | Example: + | bd mol show bd-patrol --parallel + flag --parallel -p type=bool default="false" usage="Show parallel step analysis" + +bd mol squash + use: "squash " + short: "Compress molecule execution into a digest" + hidden: false + runnable: true + long: + | Squash a molecule's ephemeral children into a single digest issue. + | + | This command collects all ephemeral child issues of a molecule (Ephemeral=true), + | generates a summary digest, and promotes the wisps to persistent by + | clearing their Wisp flag (or optionally deletes them). + | + | The squash operation: + | 1. Loads the molecule and all its children + | 2. Filters to only wisps (ephemeral issues with Ephemeral=true) + | 3. Generates a digest (summary of work done) + | 4. Creates a permanent digest issue (Ephemeral=false) + | 5. Clears Wisp flag on children (promotes to persistent) + | OR keeps them with --keep-children (default: delete) + | + | AGENT INTEGRATION: + | Use --summary to provide an AI-generated summary. This keeps bd as a pure + | tool - the calling agent (orchestrator worker, Claude Code, etc.) is responsible + | for generating intelligent summaries. Without --summary, a basic concatenation + | of child issue content is used. + | + | This is part of the wisp workflow: spawn creates wisps, + | execution happens, squash compresses the trace into an outcome (digest). + | + | Example: + | bd mol squash bd-abc123 # Squash and promote children + | bd mol squash bd-abc123 --dry-run # Preview what would be squashed + | bd mol squash bd-abc123 --keep-children # Keep wisps after digest + | bd mol squash bd-abc123 --summary "Agent-generated summary of work done" + flag --dry-run type=bool default="false" usage="Preview what would be squashed" + flag --keep-children type=bool default="false" usage="Don't delete ephemeral children after squash" + flag --summary type=string default="" usage="Agent-provided summary (bypasses auto-generation)" + +bd mol stale + use: "stale" + short: "Detect molecules with all children closed but root still open" + hidden: false + runnable: true + long: + | Detect molecules (epics with children) whose children are all closed + | with completing reasons, but the root is still open. + | + | A molecule is considered stale if: + | 1. Every child is closed with a completing reason (not duplicate/wontfix/superseded) + | 2. Root issue is still open + | 3. Not assigned to anyone (optional, use --unassigned) + | 4. Is blocking other work (optional, use --blocking) + | + | Review whether epic scope is actually finished before closing — child + | closure alone does not prove the epic's stated work is done (GH#5026). + | + | Examples: + | bd mol stale # List all stale molecules + | bd mol stale --json # Machine-readable output + | bd mol stale --blocking # Only show those blocking other work + | bd mol stale --unassigned # Only show unassigned molecules + | bd mol stale --all # Include molecules with 0 children + flag --all type=bool default="false" usage="Include molecules with 0 children" + flag --blocking type=bool default="false" usage="Only show molecules blocking other work" + flag --unassigned type=bool default="false" usage="Only show unassigned molecules" + +bd mol wisp + use: "wisp [proto-id]" + short: "Create or manage wisps (ephemeral molecules)" + hidden: false + runnable: true + long: + | Create or manage wisps - EPHEMERAL molecules for operational workflows. + | + | When called with a proto-id argument, creates a wisp from that proto. + | When called with a subcommand (list, gc), manages existing wisps. + | + | Wisps are issues with Ephemeral=true in the main database. They're stored + | locally but NOT synced via git. + | + | WHEN TO USE WISP vs POUR: + | wisp (vapor): Ephemeral work that auto-cleans up + | - Release workflows (one-time execution) + | - Operational loops and recurring cycles + | - Health checks and diagnostics + | - Any operational workflow without audit value + | + | pour (liquid): Persistent work that needs audit trail + | - Feature implementations spanning multiple sessions + | - Work you may need to reference later + | - Anything worth preserving in git history + | + | TIP: Formulas can specify phase:"vapor" to recommend wisp usage. + | If you use pour on a vapor-phase formula, you'll get a warning. + | + | The wisp lifecycle: + | 1. Create: bd mol wisp or bd create --ephemeral + | 2. Execute: Normal bd operations work on wisp issues + | 3. Squash: bd mol squash (clears Ephemeral flag, promotes to persistent) + | 4. Or burn: bd mol burn (deletes without creating digest) + | + | Examples: + | bd mol wisp beads-release --var version=1.0 # Release workflow + | bd mol wisp mol-my-workflow # Ephemeral operational cycle + | bd mol wisp list # List all wisps + | bd mol wisp gc # Garbage collect old wisps + | + | Subcommands: + | list List all wisps in current context + | gc Garbage collect orphaned wisps + flag --dry-run type=bool default="false" usage="Preview what would be created" + flag --root-only type=bool default="false" usage="Create only the root issue (no child step issues)" + flag --var type=stringArray default="[]" usage="Variable substitution (key=value); a name the proto cannot consume is an error" + +bd mol wisp create + use: "create " + short: "Instantiate a proto as a wisp (solid -> vapor)" + hidden: false + runnable: true + long: + | Create a wisp from a proto - sublimation from solid to vapor. + | + | This is the chemistry-inspired command for creating ephemeral work from templates. + | The resulting wisp is stored in the main database with Ephemeral=true and NOT synced via git. + | + | Phase transition: Proto (solid) -> Wisp (vapor) + | + | Use wisp for: + | - Operational loops and recurring cycles + | - Health checks and monitoring + | - One-shot orchestration runs + | - Routine operations with no audit value + | + | The wisp will: + | - Be stored in main database with Ephemeral=true flag + | - NOT be synced via git + | - Either evaporate (burn) or condense to digest (squash) + | + | Examples: + | bd mol wisp create mol-patrol # Ephemeral patrol cycle + | bd mol wisp create mol-health-check # One-time health check + | bd mol wisp create mol-diagnostics --var target=db # Diagnostic run + flag --dry-run type=bool default="false" usage="Preview what would be created" + flag --root-only type=bool default="false" usage="Create only the root issue (no child step issues)" + flag --var type=stringArray default="[]" usage="Variable substitution (key=value); a name the proto cannot consume is an error" + +bd mol wisp gc + use: "gc" + short: "Garbage collect old/abandoned wisps" + hidden: false + runnable: true + long: + | Garbage collect old or abandoned wisps from the database. + | + | A wisp is considered abandoned if: + | - It hasn't been updated in --age duration and is not closed + | - AND it is not live work: blocked steps (waiting on a dependency), pinned + | beads, and any step whose status category is wip (in_progress, blocked, + | hooked) or frozen (deferred, pinned) are never reclaimed by age, no matter + | how long they have been waiting (GH#4394). Custom statuses count by their + | configured category, so only plain open (active) and closed (done) steps + | are age-reclaimable. If the blocked set or the custom-status list cannot be + | read, the GC aborts rather than risk reclaiming live steps. + | + | Wisps that depend on an abandoned wisp are deleted with it, closed ones + | included even without --all, but only if they too haven't been updated in + | --age duration and are not live work. A dependent that is too young is + | spared and loses its edge to the deleted wisp: an open one is collected once + | it ages out, a closed one only by --all, --closed or 'bd purge'. + | + | Abandoned wisps are deleted without creating a digest. Use 'bd mol squash' + | if you want to preserve a summary before garbage collection. + | + | Use --closed to purge ALL closed wisps (regardless of age). This is the + | fastest way to reclaim space from accumulated wisp bloat. Safe by default: + | requires --force to actually delete. + | + | Note: This uses time-based cleanup, appropriate for ephemeral wisps. + | For graph-pressure staleness detection (blocking other work), see 'bd mol stale'. + | + | Examples: + | bd mol wisp gc # Clean abandoned wisps (default: 1h threshold) + | bd mol wisp gc --dry-run # Preview what would be cleaned + | bd mol wisp gc --age 24h # Custom age threshold + | bd mol wisp gc --all # Also clean closed wisps older than threshold + | bd mol wisp gc --closed # Preview closed wisp deletion + | bd mol wisp gc --closed --force # Delete all closed wisps + | bd mol wisp gc --closed --dry-run # Explicit dry-run (same as no --force) + | bd mol wisp gc --exclude-type agent,rig # Protect agent and rig wisps from GC + | bd mol wisp gc --closed --force --exclude-type mol # Delete closed wisps except mol type + flag --age type=string default="1h" usage="Age threshold for abandoned wisp detection" + flag --all type=bool default="false" usage="Also clean closed wisps older than threshold" + flag --closed type=bool default="false" usage="Delete all closed wisps (ignores --age threshold)" + flag --dry-run type=bool default="false" usage="Preview what would be cleaned" + flag --exclude-type type=stringSlice default="[]" usage="Exclude wisps of these types from GC (comma-separated, e.g., agent,rig)" + flag --force -f type=bool default="false" usage="Actually delete (default: preview only)" + +bd mol wisp list + use: "list" + short: "List all wisps in current context" + hidden: false + runnable: true + long: + | List all wisps (ephemeral molecules) in the current context. + | + | Wisps are issues with Ephemeral=true in the main database. They are stored + | locally but not synced via git. + | + | The list shows: + | - ID: Issue ID of the wisp + | - Title: Wisp title + | - Status: Current status (open, in_progress, closed) + | - Started: When the wisp was created + | - Updated: Last modification time + | + | Old wisp detection: + | - Old wisps haven't been updated in 24+ hours + | - Use 'bd mol wisp gc' to clean up old/abandoned wisps + | + | Examples: + | bd mol wisp list # List all wisps + | bd mol wisp list --json # JSON output for programmatic use + | bd mol wisp list --all # Include closed wisps + flag --all type=bool default="false" usage="Include closed wisps" + flag --type type=string default="" usage="Filter by issue type (e.g., agent, task, patrol)" + +bd note + use: "note [text...]" + short: "Append a note to an issue" + group: "issues" + hidden: false + runnable: true + long: + | Append a note to an issue's notes field. + | + | Shorthand for 'bd update --append-notes "text"'. + | + | Examples: + | bd note gt-abc "Fixed the flaky test" + | bd note gt-abc Fixed the flaky test + | echo "note from pipe" | bd note gt-abc --stdin + | bd note gt-abc --file notes.txt + | + | Note: "note" has NO subcommands — it only appends. + | To read notes on an issue, use: bd show + flag --file type=string default="" annotations={cobra_annotation_mutually_exclusive=["stdin file"]} usage="Read note text from file" + flag --stdin type=bool default="false" annotations={cobra_annotation_mutually_exclusive=["stdin file"]} usage="Read note text from stdin" + +bd notion + use: "notion" + short: "Notion integration commands" + hidden: false + runnable: false + long: + | Commands for syncing issues between beads and Notion. + +bd notion connect + use: "connect" + short: "Connect bd to an existing Notion database or data source" + hidden: false + runnable: true + flag --url type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Existing Notion database or data source URL" + +bd notion init + use: "init" + short: "Create a dedicated Beads database in Notion" + hidden: false + runnable: true + flag --parent type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Parent page ID" + flag --title type=string default="Beads Issues" usage="Database title" + +bd notion pull + use: "pull [refs...]" + short: "Pull specific items from Notion" + hidden: false + runnable: true + long: + | Pull one or more items from Notion. + | + | Accepts bead IDs or external references as positional arguments. + | Equivalent to: bd notion sync --pull --issues + flag --dry-run type=bool default="false" usage="Preview pull without making changes" + +bd notion push + use: "push [bead-ids...]" + short: "Push specific beads to Notion" + hidden: false + runnable: true + long: + | Push one or more beads issues to Notion. + | + | Accepts bead IDs as positional arguments. + | Equivalent to: bd notion sync --push --issues + flag --dry-run type=bool default="false" usage="Preview push without making changes" + +bd notion status + use: "status" + short: "Show Notion sync status" + hidden: false + runnable: true + +bd notion sync + use: "sync" + short: "Sync issues with Notion" + hidden: false + runnable: true + long: + | Synchronize issues between beads and Notion. + | + | By default this performs bidirectional sync. Use --pull or --push to limit direction. + flag --create-only type=bool default="false" usage="Only create missing remote pages, do not update existing ones" + flag --dry-run type=bool default="false" usage="Preview changes without making mutations" + flag --issues type=string default="" usage="Comma-separated bead IDs to sync selectively (e.g., bd-abc,bd-def). Mutually exclusive with --parent." + flag --parent type=string default="" usage="Limit push to this bead and its descendants (push only). Mutually exclusive with --issues." + flag --prefer-local type=bool default="false" usage="On conflict, keep the local beads version" + flag --prefer-notion type=bool default="false" usage="On conflict, use the Notion version" + flag --pull type=bool default="false" usage="Only pull issues from Notion" + flag --push type=bool default="false" usage="Only push issues to Notion" + flag --state type=string default="all" usage="Issue state to sync: open, closed, or all" + +bd onboard + use: "onboard" + short: "Display minimal snippet for agent instructions file" + group: "setup" + hidden: false + runnable: true + long: + | Display a minimal snippet to add to your agent instructions file for bd integration. + | + | By default, the agent instructions file is AGENTS.md. Use 'bd init --agents-file' + | to configure a different filename (e.g. BEADS.md). + | + | This outputs a small (~10 line) snippet that points to 'bd prime' for full + | workflow context. This is the same minimal profile that 'bd init' generates + | by default. This approach: + | + | • Keeps your agent file lean (doesn't bloat with instructions) + | • bd prime provides dynamic, always-current workflow details + | • Hooks auto-inject bd prime at session start + | + | For agents or environments that do not auto-inject hook output, use + | 'bd init --agents-profile=full' to embed the complete command reference. + +bd orphans + use: "orphans" + short: "Identify orphaned issues (referenced in commits but still open)" + hidden: false + runnable: true + long: + | Identify orphaned issues - issues that are referenced in commit messages but remain open or in_progress in the database. + | + | This helps identify work that has been implemented but not formally closed. + | + | Examples: + | bd orphans # Show orphaned issues + | bd orphans --json # Machine-readable output + | bd orphans --details # Show full commit information + | bd orphans --fix # Close orphaned issues with confirmation + | bd orphans --label theme:personal # Only orphans with this label + | bd orphans --label-any theme:personal,theme:ventures # Orphans with either label + flag --details type=bool default="false" usage="Show full commit information" + flag --fix -f type=bool default="false" usage="Close orphaned issues with confirmation" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL). Can combine with --label-any" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE). Can combine with --label" + +bd ping + use: "ping" + short: "Check database connectivity" + group: "maint" + hidden: false + runnable: true + long: + | Lightweight health check that confirms bd can reach its database. + | + | Steps: + | 1. Resolve the .beads workspace + | 2. Open the store (embedded or server) + | 3. Run a trivial query (one issue row; a remote backend reads its + | authenticated server context instead) + | 4. Report timing + | + | Exit 0 on success, exit 1 on failure. + | + | Examples: + | bd ping # Quick connectivity check + | bd ping --json # Structured output for automation + +bd preflight + use: "preflight" + short: "Show PR readiness checklist" + group: "maint" + hidden: false + runnable: true + long: + | Display a checklist of common pre-PR checks for contributors. + | + | This command helps catch common issues before pushing to CI: + | - Tests not run locally + | - Lint errors + | - Unformatted Go files + | - .beads/issues.jsonl pollution + | - Stale nix vendorHash + | - Version mismatches + | + | Examples: + | bd preflight # Show checklist + | bd preflight --check # Run checks automatically + | bd preflight --check --json # JSON output for programmatic use + | bd preflight --check --skip-lint # Explicitly skip lint check + | + flag --check type=bool default="false" usage="Run checks automatically" + flag --fix type=bool default="false" usage="Auto-fix issues where possible (vendorHash)" + flag --skip-lint type=bool default="false" usage="Skip lint check explicitly" + +bd prime + use: "prime" + short: "Output AI-optimized workflow context" + group: "setup" + hidden: false + runnable: true + long: + | Output essential Beads workflow context in AI-optimized markdown format. + | + | Automatically detects if MCP server is active and adapts output: + | - MCP mode: Brief workflow reminders (~50 tokens) + | - CLI mode: Full command reference (~1-2k tokens) + | + | Designed for Claude Code, Gemini CLI, and Codex SessionStart hooks to prevent + | agents from forgetting bd workflow after context compaction. + | + | Config options: + | - no-git-ops: When true, outputs stealth mode (no git commands in session close protocol). + | Set via: bd config set no-git-ops true + | Useful when you want to control when commits happen manually. + | - agent.profile: Explicit policy profile for git/commit authority wording + | (conservative | minimal | team-maintainer; default conservative). + | Set via: bd config set agent.profile team-maintainer + | Or per-session: BD_AGENT_PROFILE=team-maintainer (env var takes precedence). + | See docs/getting-started/ide-setup.md#policy-profiles for what each profile means. + | - prime.disabled: When true, bd prime prints nothing, as it does outside a + | workspace (an empty envelope under --hook-json), whatever its other flags. + | For sessions whose host briefs the agent itself but that inherit a + | project's bd prime hook, such as an orchestrator's agent sessions. + | Set per-session: BD_PRIME_DISABLED=true. Or: bd config set prime.disabled true + | + | Workflow customization: + | - Place a PRIME.md file to override the default workflow text. Checked in this + | order, once bd resolves a workspace (outside one, prime emits no content): + | (1) .beads/PRIME.md relative to the current directory (the -C target + | when -C is set); + | (2) PRIME.md in the .beads directory bd resolves for this workspace + | (honors $BEADS_DIR, which -C overrides; a redirected .beads is + | followed); + | (3) the global PRIME.md in bd's user config dir: + | ~/.config/beads/ on Linux ($XDG_CONFIG_HOME/beads/ if set), + | ~/Library/Application Support/beads/ on macOS, + | %AppData%\beads\ on Windows. + | - Persistent memories (from bd remember) are still appended so memory + | injection keeps working under a custom template. + | - Use --export to dump the default content for customization. + | - Use --memories-only for hook contexts that should inject only persistent memories; this returns only the memories section even when a custom PRIME.md is present. + | - Use --no-memories to omit the persistent memories section (useful when the memories section is large and would dominate a context budget). --memories-only takes precedence if both are set. + | + | Memory injection caps: + | Large memory sets can exceed what a session-start hook host will ingest, + | and hosts truncate silently. Cap what prime injects with --max-memories N + | and/or --max-memory-chars N (or the prime.max-memories / + | prime.max-memory-chars config keys; an explicit flag wins, and an explicit + | 0 forces unlimited). Caps apply at whole-memory boundaries, at least one + | memory is always emitted, and a banner ahead of the entries reports how + | many were elided and how to browse the rest with bd memories. + | --max-memory-chars caps the total bytes of the injected memory entries; + | the section header and elision banner are excluded from the budget. + flag --export type=bool default="false" usage="Output default content (ignores PRIME.md override)" + flag --full type=bool default="false" usage="Force full CLI output (ignore MCP detection)" + flag --hook-json type=bool default="false" usage="Wrap output in the SessionStart hook JSON envelope (Claude Code, Gemini CLI, Codex)" + flag --max-memories type=int default="0" usage="Cap injected persistent memories to N entries (0 = unlimited; falls back to the prime.max-memories config key)" + flag --max-memory-chars type=int default="0" usage="Cap the total bytes of injected memory entries, at whole-memory boundaries; section header and banner are not counted (0 = unlimited; falls back to the prime.max-memory-chars config key)" + flag --mcp type=bool default="false" usage="Force MCP mode (minimal output)" + flag --memories-only type=bool default="false" usage="Output only persistent memories for compact hook contexts" + flag --no-memories type=bool default="false" usage="Omit the persistent memories section (ignored when --memories-only is set, which wins)" + flag --stealth type=bool default="false" usage="Stealth mode (no git operations, flush only)" + +bd priority + use: "priority " + short: "Set the priority of an issue" + group: "issues" + hidden: false + runnable: true + long: + | Set the priority of an issue. + | + | Shorthand for 'bd update --priority '. + | + | Priority levels: + | 0 - Critical (security, data loss, broken builds) + | 1 - High (major features, important bugs) + | 2 - Medium (default) + | 3 - Low (polish, optimization) + | 4 - Backlog (future ideas) + | + | Examples: + | bd priority bd-123 0 # Critical + | bd priority bd-123 2 # Medium + +bd promote + use: "promote " + short: "Promote a wisp to a permanent bead" + group: "issues" + hidden: false + runnable: true + long: + | Promote a wisp (ephemeral issue) to a permanent bead. + | + | This copies the issue from the wisps table (dolt_ignored) to the permanent + | issues table (Dolt-versioned), preserving labels, dependencies, events, and + | comments. The original ID is preserved so all links keep working. + | + | A comment is added recording the promotion and optional reason. + | + | Examples: + | bd promote bd-wisp-abc123 + | bd promote bd-wisp-abc123 --reason "Worth tracking long-term" + flag --reason -r type=string default="" usage="Reason for promotion" + +bd provenance + use: "provenance" + short: "Append-only provenance event log" + group: "issues" + hidden: false + runnable: false + long: + | Record and read provenance events: typed bindings from an issue to an + | opaque external artifact (a git SHA, PR, work-id, transcript, or branch). + | + | The log is append-only — there is no update or delete. bd never interprets the + | actor or ref; only kind and ref-kind are structurally validated. Recording is + | idempotent on a deterministic id, so a producer firing twice is harmless. + +bd provenance by-ref + use: "by-ref " + short: "List provenance events bound to a ref" + hidden: false + runnable: true + +bd provenance log + use: "log " + short: "List provenance events for an issue" + hidden: false + runnable: true + flag --kind type=string default="" usage="filter by kind (optional)" + +bd provenance record + use: "record --issue --kind --source " + short: "Record a provenance event (idempotent)" + hidden: false + runnable: true + long: + | Record a provenance event. The event is appended idempotently: a + | deterministic id is computed from source:issue:kind:(ref or --at), so re-running + | the same record is a no-op. + | + | An event recorded without --ref requires --at so the id is caller-owned. + flag --actor type=string default="" usage="opaque actor identifier (optional)" + flag --at type=string default="" usage="event-time as RFC3339 (required for ref-less kinds)" + flag --issue type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="issue id (required)" + flag --kind type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="event kind: cut|claim|suspend|resume|handoff|commit|land|used (required)" + flag --payload type=string default="" usage="opaque payload, e.g. JSON (optional)" + flag --ref type=string default="" usage="opaque external reference, e.g. a SHA or PR url (optional)" + flag --ref-kind type=string default="" usage="ref kind: git-sha|pr|work-id|transcript|branch (optional)" + flag --source type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="producer of the event, e.g. git-hook, orchestrator (required)" + +bd prune + use: "prune" + short: "Delete old closed beads to reclaim space and shrink exports" + group: "maint" + hidden: false + runnable: true + long: + | Permanently delete closed non-ephemeral beads and their associated data. + | + | Use this to trim closed regular beads (tasks, features, bugs, chores, etc.) + | that are no longer useful. The common case is a long-lived repo where + | closed work has piled up and is bloating auto-export or slowing queries. + | + | Requires --older-than or --pattern. The flag is a safety gate — without + | it, a muscle-memory `--force` could wipe every closed bead in the repo. + | Use `--pattern '*'` if you really do want to sweep everything closed. + | + | Deletes: issues, dependencies, labels, events, and comments for matching beads. + | Skips: pinned beads (protected), open/in-progress beads, and ephemeral beads. + | + | Also skips closed beads whose ID appears in the description, notes, or + | comments of any open / in-progress bead. This protects ADR / decision / + | verification trails that downstream beads still cite. Use + | --ignore-references to override (e.g., when bulk-decommissioning a + | retired label across the rig). + | + | To delete closed ephemeral beads (wisps, transient molecules) use + | `bd purge` instead. + | + | For full Dolt storage reclaim after deleting many rows, follow with `bd flatten` + | so history can be collapsed and old chunks can be garbage-collected. + | + | EXAMPLES: + | bd prune --older-than 30d # Preview closed beads >30d old + | bd prune --older-than 30d --force # Delete them + | bd prune --older-than 90d --dry-run # Detailed preview with stats + | bd prune --pattern "*" --force # Delete all closed regular beads + | bd prune --pattern "gm-temp-*" --force # Scope to a pattern + flag --dry-run type=bool default="false" usage="Preview what would be pruned with stats" + flag --force -f type=bool default="false" usage="Actually prune (without this, shows preview)" + flag --ignore-references type=bool default="false" usage="Delete closed beads even when referenced by open beads (use with care; see --help for details)" + flag --older-than type=string default="" usage="Only prune beads closed more than N ago (e.g., 30d, 2w, 60, 36h)" + flag --pattern type=string default="" usage="Only prune beads matching ID glob pattern (e.g., 'gm-old-*')" + +bd purge + use: "purge" + short: "Delete closed ephemeral beads to reclaim space" + group: "maint" + hidden: false + runnable: true + long: + | Permanently delete closed ephemeral beads and their associated data. + | + | Closed ephemeral beads (wisps, transient molecules) accumulate rapidly and + | have no value once closed. This command removes them to reclaim storage. + | + | Deletes: issues, dependencies, labels, events, and comments for matching beads. + | Skips: pinned beads, and closed beads a live bead still depends on through a + | parent-child, tracks or blocks edge (a closed molecule root whose step is open, + | a closed bead a live convoy tracks). Live means any status that is not done. + | + | --wisps-plane selects by storage plane instead: every closed row stored in the + | wisps table, including --no-history beads, which the default ephemeral + | selection leaves to `bd prune`. It reaches durable-tier rows, so it requires + | --older-than or --pattern, like `bd prune`. + | + | --limit N caps one run at N beads, oldest-closed first, so a large backlog can + | be drained in bounded transactions; --json then reports "remaining" and + | "has_more". Loop while has_more is true. + | + | --older-than takes days (7, 7d), weeks (2w) or a duration with hour precision + | (36h, 168h, 90m). + | + | To delete closed non-ephemeral beads (regular tasks, features, bugs, etc.) + | use `bd prune` instead. + | + | For full Dolt storage reclaim after deleting many rows, follow with `bd flatten` + | so history can be collapsed and old chunks can be garbage-collected. + | + | EXAMPLES: + | bd purge # Preview what would be purged + | bd purge --force # Delete all closed ephemeral beads + | bd purge --older-than 7d --force # Only purge items closed 7+ days ago + | bd purge --pattern "*-wisp-*" # Only purge matching ID pattern + | bd purge --older-than 36h --force # Hour precision: closed 36+ hours ago + | bd purge --wisps-plane --older-than 168h --force + | # Every closed wisps-table row, incl. no-history + | bd purge --dry-run # Detailed preview with stats + flag --dry-run type=bool default="false" usage="Preview what would be purged with stats" + flag --force -f type=bool default="false" usage="Actually purge (without this, shows preview)" + flag --limit type=int default="0" usage="Purge at most N beads this run, oldest-closed first (0 = no limit); --json reports remaining/has_more" + flag --older-than type=string default="" usage="Only purge beads closed more than N ago (e.g., 7d, 2w, 30, 36h)" + flag --pattern type=string default="" usage="Only purge beads matching ID glob pattern (e.g., *-wisp-*)" + flag --wisps-plane type=bool default="false" usage="Select every closed row in the wisps table, including --no-history beads (requires --older-than or --pattern)" + +bd q + use: "q [title]" + short: "Quick capture: create issue and output only ID" + group: "issues" + hidden: false + runnable: true + long: + | Quick capture creates an issue and outputs only the issue ID. + | Designed for scripting and AI agent integration. + | + | Example: + | bd q "Fix login bug" # Outputs: bd-a1b2 + | ISSUE=$(bd q "New feature") # Capture ID in variable + | bd q "Task" | xargs bd show # Pipe to other commands + | bd q "Subtask" --parent=bd-a1b2 # Hierarchical child (outputs: bd-a1b2.1) + flag --labels -l type=stringSlice default="[]" usage="Labels" + flag --parent type=string default="" usage="Parent issue ID for hierarchical child (e.g., 'bd-a3f8e9')" + flag --priority -p type=string default="2" usage="Priority (0-4 or P0-P4)" + flag --type -t type=string default="task" usage="Issue type" + +bd query + use: "query [expression]" + short: "Query issues using a simple query language" + group: "issues" + hidden: false + runnable: true + long: + | Query issues using a simple query language that supports compound filters, + | boolean operators, and date-relative expressions. + | + | The query language enables complex filtering that would otherwise require + | multiple flags or piping through jq. + | + | Syntax: + | field=value Equality comparison + | field!=value Inequality comparison + | field>value Greater than + | field>=value Greater than or equal + | field1" + | bd query "status=open AND priority<=2 AND updated>7d" + | bd query "(status=open OR status=blocked) AND priority<2" + | bd query "type=bug AND label=urgent" + | bd query "NOT status=closed" + | bd query "assignee=none AND type=task" + | bd query "created>30d AND status!=closed" + | bd query "label=frontend OR label=backend" + | bd query "title=authentication AND priority=0" + flag --all -a type=bool default="false" usage="Include closed issues (default: exclude closed)" + flag --limit -n type=int default="50" usage="Limit results (default: 50, 0 = unlimited)" + flag --long type=bool default="false" usage="Show detailed multi-line output for each issue" + flag --offset type=int default="0" usage="Skip the first N matching results (0-based). Only supported under --proxied-server." + flag --parse-only type=bool default="false" usage="Only parse the query and show the AST (for debugging)" + flag --reverse -r type=bool default="false" usage="Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first)" + flag --sort type=string default="" usage="Sort by field: priority, created, updated, closed, status, id, title, type, assignee" + +bd quickstart + use: "quickstart" + short: "Quick start guide for bd" + group: "setup" + hidden: false + runnable: true + long: + | Display a quick start guide showing common bd workflows and patterns. + +bd ready + use: "ready" + short: "Show ready work (open, no active blockers)" + hidden: false + runnable: true + long: + | Show ready work (open issues with no active blockers). + | + | Excludes in_progress, blocked, deferred, and hooked issues. This uses the + | GetReadyWork API which applies blocker-aware semantics to find truly claimable work. + | + | Note: 'bd list --ready' uses the same blocker-aware ready-work semantics. + | + | Use --mol to filter to a specific molecule's steps: + | bd ready --mol bd-patrol # Show ready steps within molecule + | + | Use --gated to find molecules ready for gate-resume dispatch: + | bd ready --gated # Find molecules where a gate closed + | + | Use --claim to atomically claim the first ready issue matching the filters: + | bd ready --claim --json + | + | This is useful for agents executing molecules to see which steps can run next. + flag --assignee -a type=string default="" usage="Filter by assignee" + flag --brief type=bool default="false" usage="Omit the free-form text (description, design, acceptance criteria, notes, payload, waiters) from each row. Filters that read those fields still select on them. An omitted field is indistinguishable from an empty one; fetch a whole issue with bd show. Requires --json, and cannot be combined with --claim, --gated, --mol or --explain." + flag --claim type=bool default="false" usage="Atomically claim the first ready issue matching the filters" + flag --exclude-label type=stringSlice default="[]" usage="Exclude issues that have ANY of these labels" + flag --exclude-type type=stringSlice default="[]" usage="Exclude issue types from results (comma-separated or repeatable, e.g., --exclude-type=convoy,epic)" + flag --explain type=bool default="false" usage="Show dependency-aware reasoning for why issues are ready or blocked" + flag --flat type=bool default="false" usage="Alias for --plain, spelled the way bd list spells it" + flag --gated type=bool default="false" usage="Find molecules ready for gate-resume dispatch" + flag --has-metadata-key type=string default="" usage="Filter issues that have this metadata key set" + flag --include-deferred type=bool default="false" usage="Include issues with future defer_until timestamps" + flag --include-ephemeral type=bool default="false" usage="Include ephemeral issues (wisps) in results" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL). Can combine with --label-any" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE). Can combine with --label" + flag --label-pattern type=string default="" usage="Filter by label glob pattern (e.g., 'tech-*' matches tech-debt, tech-legacy)" + flag --label-regex type=string default="" usage="Filter by label regex pattern (e.g., 'tech-(debt|legacy)')" + flag --limit -n type=int default="100" usage="Maximum issues to show (use 0 for unlimited)" + flag --max-rows type=int default="0" usage="Hard upper bound on rows fetched from storage. Returns a non-zero exit (code 2) and an error to stderr if exceeded. 0 disables (the default). Overrides BEADS_MAX_ROWS for this invocation. Useful in CI/agent rigs that want a circuit breaker against pathological queries. Not supported under --proxied-server: an explicit --max-rows or BEADS_MAX_ROWS cap errors out rather than silently going unenforced." + flag --metadata-field type=stringArray default="[]" usage="Filter by metadata field (key=value, repeatable)" + flag --mol type=string default="" usage="Filter to steps within a specific molecule" + flag --mol-type type=string default="" usage="Filter by molecule type: swarm, patrol, or work" + flag --offset type=int default="0" usage="Skip the first N matching results (0-based). Only supported under --proxied-server." + flag --parent type=string default="" usage="Filter to descendants of this bead/epic" + flag --plain type=bool default="false" usage="Display issues as a plain numbered list" + flag --pretty type=bool default="true" usage="Display issues in a tree format with status/priority symbols" + flag --priority -p type=int default="0" usage="Filter by priority" + flag --sort -s type=string default="priority" usage="Sort policy: priority (default), hybrid, oldest" + flag --type -t type=string default="" usage="Filter by issue type (task, bug, feature, epic, decision, merge-request). Aliases: mr→merge-request, feat→feature, mol→molecule, dec/adr→decision" + flag --unassigned -u type=bool default="false" usage="Show only unassigned issues" + +bd recall + use: "recall " + short: "Retrieve a specific memory" + group: "setup" + hidden: false + runnable: true + long: + | Retrieve the full content of a memory by its key. + | + | Examples: + | bd recall dolt-phantoms + | bd recall auth-jwt + +bd reclaim + use: "reclaim" + short: "Revert stale-lease in_progress issues back to ready (dead-worker recovery)" + group: "issues" + hidden: false + runnable: true + long: + | Revert in_progress issues whose lease has gone stale back to ready. + | + | When a worker claims an issue it takes a lease that expires after a TTL, kept + | alive by 'bd heartbeat'. A worker that dies stops heartbeating, so its lease + | expires and its issue would otherwise stay in_progress forever. reclaim is the + | reaper: it finds in_progress issues whose lease expired more than --older-than + | ago, clears the assignee, and sets them back to open so another worker can + | claim them. The previous owner's stale lease is recorded as a recovery event. + | + | --older-than is a grace window past lease expiry: only leases that expired at + | least this long ago are reclaimed, so a worker briefly paused (GC, clock skew) + | is not robbed of live work. Run it from a supervisor on a timer with a window + | of roughly 2× the claim TTL. + | + | By default reclaim covers every stale lease THIS replica granted. The scope + | filters below narrow it further, using the same label surface claiming is + | scoped by (--label / --label-any / --exclude-label), plus --assignee and --id. + | Filters AND-combine and never widen the set: a reclaimed lease must still be + | stale. + | + | Replicas and leases (federated deployments) + | ------------------------------------------- + | A lease is only meaningful on the replica that granted it. Every other + | replica's view of the holder's liveness is stale by up to one sync interval, + | so a reaper elsewhere can revert a unit that is very much alive over there. + | reclaim therefore records the granting replica on each lease and SKIPS a lease + | another replica granted, summarizing what it declined on stderr (one line per + | run; 'bd -v' expands it to the first 20 leases individually). Reap it where it + | was granted; use --any-replica only when that replica is permanently gone (or + | when this node was renamed and its own old leases now look foreign — an + | ordinary heartbeat keeps a lease alive but does not re-home it to the node + | heartbeating it). Prefer the narrow form '--any-replica --id ': bare + | --any-replica reverts EVERY foreign stale lease, live peers included. + | + | Two invariants the guard cannot enforce for you: + | + | grace window > sync interval, and lease TTL > sync interval. + | + | A TTL or grace shorter than the cadence at which replicas exchange state is + | meaningless across the bridge — the remote view is a full interval old by + | construction. Raise the TTL/grace above the sync interval, never the reverse. + | The guard is opt-in: set BEADS_NODE_ID, or run 'bd config set node_id ' + | (which writes the per-machine ~/.config/bd/config.yaml — never commit a node_id + | to the git-tracked .beads/config.yaml, or every clone reads the same name and + | the guard goes armed-but-inert). One id per STORE, not per host: machines that + | are clients of the same dolt sql-server are ONE replica and must share one value + | or leave it unset. There is no hostname fallback — the hostname names the client + | process's machine, not the store — so an unnamed deployment keeps the old, + | unguarded behavior instead of stranding its own work. + | + | Examples: + | bd reclaim # default grace window (2× the lease TTL) + | bd reclaim --older-than 10m # reclaim leases expired >10m ago + | bd reclaim --older-than 0s # reclaim every currently-expired lease + | bd reclaim --label lane-a # only this machine's claim partition + | bd reclaim --label-any lane-a,lane-b --exclude-label pinned + | bd reclaim --assignee zelda --assignee epona # only these workers' leases + | bd reclaim --id wy-abc --id wy-def # exactly these issues + | bd reclaim --any-replica # also reap leases granted by a departed replica + flag --any-replica type=bool default="false" usage="Also reclaim leases granted by ANOTHER replica (unsafe unless that replica is gone; see 'Replicas and leases')" + flag --assignee -a type=stringSlice default="[]" usage="Only reclaim leases held by these assignees (repeatable)" + flag --exclude-label type=stringSlice default="[]" usage="Never reclaim issues carrying ANY of these labels" + flag --id type=stringSlice default="[]" usage="Only reclaim these issue IDs (repeatable)" + flag --label -l type=stringSlice default="[]" usage="Only reclaim issues with ALL these labels (AND). Can combine with --label-any" + flag --label-any type=stringSlice default="[]" usage="Only reclaim issues with AT LEAST ONE of these labels (OR). Can combine with --label" + flag --older-than type=duration default="10m0s" usage="Only reclaim leases that expired at least this long ago (grace window)" + +bd recompute-blocked + use: "recompute-blocked" + short: "Recompute is_blocked for all issues (repairs stale flags after a pull)" + group: "maint" + hidden: false + runnable: true + long: + | Recompute the denormalized is_blocked flag for every issue and wisp. + | + | is_blocked is derived from the dependency graph and maintained automatically by + | local writes and by a post-pull recompute scoped to what the merge changed. If + | that scoped recompute is skipped — a recompute that failed after its merge + | committed, or a conflicted pull resolved by hand — the flag can go stale, and a + | later pull that merges nothing will not refresh it (bd-6dnrw.37). 'bd ready' + | trusts the flag, so stale values silently hide ready work or surface blocked + | work. + | + | This command runs the full recompute unconditionally and commits the result. + | It is idempotent: on a consistent database it changes nothing. Works in every + | storage mode — embedded, server, and proxied-server (unlike 'bd doctor', which + | is server-mode only). + | + | On a workspace connected to a bd serve backend ('bd connect') it is a no-op + | that exits 0: the server maintains is_blocked on every write, and the column + | lives in the server's database, so the repair is run there, by the server's + | operator. --json then reports {"rows_corrected": 0, "maintained_by": "server"}. + | + | Examples: + | bd recompute-blocked # Repair stale is_blocked flags + | bd recompute-blocked --json # Machine-parseable {"rows_corrected": N} + +bd remember + use: "remember \"\"" + short: "Store a persistent memory" + group: "setup" + hidden: false + runnable: true + long: + | Store a memory that persists across sessions and account rotations. + | + | Memories are injected at prime time (bd prime) so you have them + | in every session without manual loading. + | + | The positional arg is the memory CONTENT (the key is auto-generated from it + | unless --key is given). As a convenience, if the arg is a bare key naming an + | existing memory, it is RECALLED instead of stored (same as 'bd recall'); + | a bare key naming nothing is refused. Use --key to store slug-like content. + | + | Corpus budget (opt-in, off by default): set the memories.budget-chars config + | key to a byte ceiling for the whole memory corpus (sum of len(key)+len(content)). + | A write that would cross it is refused, and --force overrides; a write that + | lands at or above 80% of the budget warns. With the key unset nothing changes. + | bd prime's --max-memory-chars also counts bytes, but of each memory as + | injected, formatting included, so a budget equal to that cap does not + | guarantee prime injects every memory; set the budget below it for that. + | + | Examples: + | bd remember "always run tests with -race flag" + | bd remember "Dolt phantom DBs hide in three places" --key dolt-phantoms + | bd remember "auth module uses JWT not sessions" --key auth-jwt + | bd remember dolt-phantoms # bare existing key: reads it (= bd recall) + flag --force type=bool default="false" usage="Store even when the write would cross the memories.budget-chars corpus budget (no effect when the budget is unset)" + flag --key type=string default="" usage="Explicit key for the memory (auto-generated from content if not set). If a memory with this key already exists, it will be updated in place" + +bd rename + use: "rename " + short: "Rename an issue ID" + hidden: false + runnable: true + long: + | Rename an issue from one ID to another. + | + | This updates: + | - The issue's primary ID + | - All references in other issues (descriptions, titles, notes, etc.) + | - Dependencies pointing to/from this issue + | - Labels, comments, and events + | + | Examples: + | bd rename bd-w382l bd-dolt # Rename to memorable ID + | bd rename gt-abc123 gt-auth # Use descriptive ID + | + | Note: The new ID must use a valid prefix for this database. + +bd rename-prefix + use: "rename-prefix " + short: "Rename the issue prefix for all issues in the database" + group: "maint" + hidden: false + runnable: true + long: + | Rename the issue prefix for all issues in the database. + | This will update all issue IDs and all text references across all fields. + | + | USE CASES: + | - Shortening long prefixes (e.g., 'knowledge-work-' → 'kw-') + | - Rebranding project naming conventions + | - Consolidating multiple prefixes after database corruption + | - Migrating to team naming standards + | + | Prefix validation rules: + | - Allowed characters: lowercase letters, numbers, hyphens + | - Must start with a letter + | - Must end with a hyphen (e.g., 'kw-', 'work-') + | - Cannot be empty or just a hyphen + | + | Multiple prefix detection and repair: + | If issues have multiple prefixes (corrupted database), use --repair to consolidate them. + | The --repair flag will rename all issues with incorrect prefixes to the new prefix, + | preserving issues that already have the correct prefix. + | + | EXAMPLES: + | bd rename-prefix kw- # Rename from 'knowledge-work-' to 'kw-' + | bd rename-prefix mtg- --repair # Consolidate multiple prefixes into 'mtg-' + | bd rename-prefix team- --dry-run # Preview changes without applying + | + | NOTE: This is a rare operation. Most users never need this command. + flag --dry-run type=bool default="false" usage="Preview changes without applying them" + flag --repair type=bool default="false" usage="Repair database with multiple prefixes by consolidating them" + +bd reopen + use: "reopen [id...]" + short: "Reopen one or more closed issues" + group: "issues" + hidden: false + runnable: true + long: + | Reopen closed issues by setting status to 'open' and clearing the closed_at timestamp. + | This is more explicit than 'bd update --status open' and emits a Reopened event. + | + | Exit codes: 1 for general failures; 13 when the failure is a stale + | --if-revision guard (the precondition no longer held, nothing was written — + | another actor won the race, so retrying the same guard is pointless). + flag --if-revision type=string default="" usage="Apply the write only if the issue's current revision equals this value (the decimal int64 from `bd show --json`'s \"revision\" field). One id only: refused before any write when combined with more than one issue id. A mismatch writes nothing and exits 13 (vs 1 for other failures)." + flag --reason -r type=string default="" usage="Reason for reopening" + +bd repo + use: "repo" + short: "Manage multiple repository configuration" + group: "advanced" + hidden: false + runnable: false + long: + | Configure and manage multiple repository support for multi-repo hydration. + | + | Multi-repo support allows hydrating issues from multiple beads repositories + | into a single database for unified cross-repo issue tracking. + | + | Configuration is stored in .beads/config.yaml under the 'repos' section: + | + | repos: + | primary: "." + | additional: + | - ~/beads-planning + | - ~/work-repo + | + | Examples: + | bd repo add ~/beads-planning # Add planning repo + | bd repo add ../other-repo # Add relative path repo + | bd repo list # Show all configured repos + | bd repo remove ~/beads-planning # Remove by path + | bd repo sync # Sync from all configured repos + +bd repo add + use: "add " + short: "Add an additional repository to sync" + hidden: false + runnable: true + long: + | Add a repository path to the repos.additional list in config.yaml. + | + | The path should point to a directory containing a .beads folder. + | Paths can be absolute or relative (they are stored as-is). + | + | This modifies .beads/config.yaml, which is version-controlled and + | shared across all clones of this repository. + +bd repo list + use: "list" + short: "List all configured repositories" + hidden: false + runnable: true + long: + | List all repositories configured in .beads/config.yaml. + | + | Shows the primary repository (always ".") and any additional + | repositories configured for hydration. + +bd repo remove + use: "remove " + short: "Remove a repository from sync configuration" + hidden: false + runnable: true + long: + | Remove a repository path from the repos.additional list in config.yaml. + | + | The path must exactly match what was added (e.g., if you added "~/foo", + | you must remove "~/foo", not "/home/user/foo"). + | + | This command also removes any previously-hydrated issues from the database + | that came from the removed repository. + +bd repo sync + use: "sync" + short: "Manually trigger multi-repo sync" + hidden: false + runnable: true + long: + | Synchronize issues from all configured additional repositories. + | + | Reads issues.jsonl from each additional repository and imports them into + | the primary database with their original prefixes and source_repo set. + | Uses mtime caching to skip repos whose JSONL hasn't changed. + | + | Also triggers Dolt push/pull if a remote is configured. + flag --verbose type=bool default="false" usage="Show detailed sync progress" + +bd restore + use: "restore " + short: "Restore the pre-compaction content of a compacted issue" + group: "sync" + hidden: false + runnable: true + long: + | Restore the pre-compaction content of a compacted issue. + | + | When an issue is compacted, its description/design/notes/acceptance criteria + | are summarized and the originals are archived to a compaction snapshot. This + | command recovers that original content. + | + | By default it is read-only: it displays the archived content without modifying + | the database. Pass --apply to write the original content back into the issue + | and step its compaction level back down. + | + | If no archived snapshot exists (e.g. the issue was compacted by an older bd + | before snapshot archiving), restore falls back to a best-effort reconstruction + | from Dolt version history, which can only be displayed, not applied. + flag --apply type=bool default="false" usage="Write the restored content back into the issue (default: display only)" + +bd rules + use: "rules" + short: "Audit and compact Claude rules" + group: "maint" + hidden: false + runnable: false + +bd rules audit + use: "audit" + short: "Scan rules for contradictions and merge opportunities" + hidden: false + runnable: true + flag --path type=string default=".claude/rules/" usage="Path to rules directory" + flag --threshold type=float64 default="0.6" usage="Jaccard similarity threshold" + +bd rules compact + use: "compact" + short: "Merge related rules into composites" + hidden: false + runnable: true + flag --auto type=bool default="false" usage="Apply audit suggestions" + flag --dry-run type=bool default="false" usage="Preview without applying" + flag --group type=stringSlice default="[]" usage="Rule names to merge" + flag --path type=string default=".claude/rules/" usage="Path to rules directory" + +bd schema + use: "schema" + short: "Print the JSON Schema for bd's --json / export output" + hidden: false + runnable: true + annotations: {bd:skip_store="1"} + long: + | Print the JSON Schema for bd's canonical output record types. + | + | The schema is reflected from the same Go structs bd serializes, so it stays in + | lockstep with the actual --json / export output. Use it to generate typed + | consumer models (datamodel-code-generator, quicktype, ...) rather than + | hand-maintaining them. + | + | bd schema | jq '.types.issue' # the issue record schema + | bd schema | jq '.types.dependency' # the dependency record schema + +bd search + use: "search [query]" + short: "Search issues by text query" + group: "issues" + hidden: false + runnable: true + long: + | Search issues across title and ID (all statuses, including closed). + | + | ID-like queries (e.g., "bd-123", "hq-319") use fast exact/prefix matching. + | Text queries search titles. Use --desc-contains for description search. + | Use --status open (etc.) to narrow; closed issues are included by default + | so "was this already filed/fixed?" cannot silently answer no. Matches + | beyond --limit are dropped status-blind, so when hunting live work in a + | large DB, narrow with --status open or raise --limit. + | + | Examples: + | bd search "authentication bug" + | bd search "login" --status open + | bd search "database" --label backend --limit 10 + | bd search --query "performance" --assignee alice + | bd search "bd-5q" # Search by partial ID (fast prefix match) + | bd search "security" --priority-min 0 --priority-max 2 + | bd search "bug" --created-after 2025-01-01 + | bd search "refactor" --status open # Only open issues + | bd search "bug" --sort priority + | bd search "task" --sort created --reverse + | bd search "api" --desc-contains "endpoint" + | bd search "cleanup" --no-assignee --no-labels + flag --assignee -a type=string default="" usage="Filter by assignee" + flag --closed-after type=string default="" usage="Filter issues closed after date (YYYY-MM-DD or RFC3339)" + flag --closed-before type=string default="" usage="Filter issues closed before date (YYYY-MM-DD or RFC3339)" + flag --created-after type=string default="" usage="Filter issues created after date (YYYY-MM-DD or RFC3339)" + flag --created-before type=string default="" usage="Filter issues created before date (YYYY-MM-DD or RFC3339)" + flag --desc-contains type=string default="" usage="Filter by description substring (case-insensitive)" + flag --empty-description type=bool default="false" usage="Filter issues with empty or missing description" + flag --external-contains type=string default="" usage="Filter by external ref substring (case-insensitive)" + flag --has-metadata-key type=string default="" usage="Filter issues that have this metadata key set" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL)" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE)" + flag --limit -n type=int default="50" usage="Limit results (default: 50)" + flag --long type=bool default="false" usage="Show detailed multi-line output for each issue" + flag --metadata-field type=stringArray default="[]" usage="Filter by metadata field (key=value, repeatable)" + flag --no-assignee type=bool default="false" usage="Filter issues with no assignee" + flag --no-labels type=bool default="false" usage="Filter issues with no labels" + flag --notes-contains type=string default="" usage="Filter by notes substring (case-insensitive)" + flag --priority-max type=string default="" usage="Filter by maximum priority (inclusive, 0-4 or P0-P4)" + flag --priority-min type=string default="" usage="Filter by minimum priority (inclusive, 0-4 or P0-P4)" + flag --query type=string default="" usage="Search query (alternative to positional argument)" + flag --reverse -r type=bool default="false" usage="Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first)" + flag --sort type=string default="" usage="Sort by field: priority, created, updated, closed, status, id, title, type, assignee" + flag --status -s type=string default="" usage="Filter by stored status (comma-separated for OR; open, in_progress, blocked, deferred, closed, all). Default searches all statuses including closed. Note: dependency-blocked issues use 'bd blocked'" + flag --type -t type=string default="" usage="Filter by type (bug, feature, task, epic, chore, decision, message, molecule, gate, spike, story, milestone); custom types require types.custom config, and an unknown type is an error. Aliases: feat/enhancement→feature, mol→molecule, dec/adr→decision, mr→merge-request" + flag --updated-after type=string default="" usage="Filter issues updated after date (YYYY-MM-DD or RFC3339)" + flag --updated-before type=string default="" usage="Filter issues updated before date (YYYY-MM-DD or RFC3339)" + +bd send-metrics + use: "send-metrics" + short: "Internal: flush queued telemetry events (spawned by bd)" + hidden: true + runnable: true + +bd serve + use: "serve" + short: "Serve the beads HTTP API over loopback" + hidden: false + runnable: true + long: + | Serve the beads HTTP API — the same work surface the CLI answers, for + | automation clients that would otherwise fork a bd subprocess per call. + | + | The wire contract is described by an OpenAPI document (/v0); GET + | /v0/beads/context reports which operations this build actually implements. + | + | DEPLOYMENT + | + | Pass an explicit port. The default 127.0.0.1:0 takes an ephemeral one, which + | is right for ad-hoc and test use — where the bound address printed on stdout + | is read immediately — but carries no mutual exclusion: two serves against one + | workspace then run side by side on different ports with no way to enumerate + | them. On a fixed port the second one fails to bind, which is the intended + | behavior. Concurrent serves are data-safe either way; claims are arbitrated + | in the SQL server. + | + | Run it under a supervisor. bd shuts down gracefully on SIGHUP as well as + | SIGINT and SIGTERM, so closing the terminal of a foreground bd serve stops it. + | + | PROBES + | + | GET /healthz is LIVENESS only: it answers from the process and never touches + | the database, so it stays green while the database is unreachable. For + | readiness use GET /v0/beads/ready?limit=1 — a real query, where 200 means + | ready and 503 means live but not ready. + | + | AUTHENTICATION + | + | Optional, and off by default on loopback — where the trust model is the + | loopback boundary itself, the same one the database behind it already relies + | on. --auth-token-file turns it on: every operation except GET /healthz then + | requires an "Authorization: Bearer " header, GET /v0/beads/context + | included, because it reports the repo root, beads directory and database name. + | + | The file holds ONE TOKEN PER LINE and every line is accepted. That is the + | rotation mechanism: write the new token alongside the old, roll the clients + | over, then delete the old line. The file is re-read while the server runs, so + | both the addition and the removal take effect within about a second and + | neither needs a restart. Write it atomically (a temp file plus rename; a + | Kubernetes secret mount already does this). + | + | There is deliberately no --auth-token flag. A credential passed as an + | argument is readable by every local user in the process listing. + | + | --allow-non-loopback REQUIRES a token file. Beyond loopback, reaching the + | address would otherwise be the whole authorization: any peer could read every + | issue and claim work as any actor. --insecure-no-auth is the explicit, + | auditable way to say you meant that anyway. + | + | The Host allowlist is what a service deployment usually trips over first. The + | DNS-rebinding check answers only to loopback spellings and the bind address, + | so a client dialing a service name gets 400 on every request; enumerate the + | names it dials with --allowed-host (repeatable). Matching is exact — no + | wildcards — and the startup log line prints the effective allowlist. + | + | WHAT THIS DOES NOT DO + | + | No TLS. Even with a token, the credential and every issue body travel in + | plaintext, so a deployment beyond loopback has to supply confidentiality + | itself — a service mesh, or a network boundary you already trust. + | + | Hooks do not fire. A hook is a user-controlled subprocess per mutation: in a + | concurrent server that is an unbounded latency multiplier and an orphaned + | child at shutdown, and its working-directory-derived hook lookup is + | meaningless in a server process. A CLI claim runs on_update; an HTTP claim + | does not. + | + | The per-command auto-commit machinery does not run. Durability is per request: + | a successful claim commits inside its own transaction, exactly as a proxied + | CLI claim does today. + | + | An actor on an HTTP request is caller-asserted provenance for the audit trail, + | not authenticated identity — the same thing it has always been on the CLI, + | where any local process can pass any --actor. + | + | It does not run under --readonly, and refuses to start rather than binding. + | Every server it binds publishes the issue-claim operation, and the capability + | set it advertises is a property of the build rather than of the flags on the + | process that started it — so a read-only server would advertise a write it + | could never land. + | + | DESTRUCTIVE OPERATIONS + | + | POST /v0/beads/issues:sweep deletes closed beads in bulk — the operation + | behind bd purge and bd prune — and nothing it deletes comes back. It shares + | the library surface those commands call, so it inherits their guards: pinned + | beads are never swept, and a durable sweep with neither a cutoff nor an id + | pattern is refused rather than clearing every closed bead in the workspace. + | Combined with the trust model above, that means anyone who can reach this + | address can erase closed work; bind it accordingly. + flag --addr type=string default="127.0.0.1:0" usage="Address to bind as IP:PORT; the host must be a numeric IP literal, and port 0 takes an ephemeral port" + flag --allow-non-loopback type=bool default="false" usage="Permit a bind beyond loopback. Requires --auth-token-file, since reaching the address would otherwise be the whole authorization" + flag --allowed-host type=stringArray default="[]" usage="Additional Host header value to answer to, e.g. a service DNS name. Repeatable; matched exactly, with no wildcards" + flag --auth-token-file type=string default="" usage="Require an Authorization: Bearer token from this file, one token per line, all accepted. Re-read while running, so rewriting it rotates tokens without a restart (env BEADS_SERVE_TOKEN_FILE)" + flag --insecure-no-auth type=bool default="false" usage="Serve a non-loopback bind with NO authentication. Every peer that can reach the address gets full read and claim access" + flag --large-apply-ceiling type=duration default="5m0s" usage="Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long plus 5s, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md" + +bd set-state + use: "set-state =" + short: "Set operational state (creates event + updates label)" + group: "issues" + hidden: false + runnable: true + long: + | Atomically set operational state on an issue. + | + | This command: + | 1. Creates an event bead recording the state change (source of truth) + | 2. Removes any existing label for the dimension + | 3. Adds the new dimension:value label (fast lookup cache) + | + | State labels follow the convention :, for example: + | patrol:active, patrol:muted + | mode:normal, mode:degraded + | health:healthy, health:failing + | + | Examples: + | bd set-state agent-abc patrol=muted --reason "Investigating stuck worker" + | bd set-state agent-abc mode=degraded --reason "High error rate detected" + | bd set-state agent-abc health=healthy + | + | The --reason flag provides context for the event bead (recommended). + flag --reason type=string default="" usage="Reason for the state change (recorded in event)" + +bd setup + use: "setup [recipe]" + short: "Setup integration with AI editors" + group: "setup" + hidden: false + runnable: true + long: + | Setup integration files for AI editors and coding assistants. + | + | Recipes define where beads workflow instructions are written. Built-in recipes + | include cursor, claude, copilot, gemini, aider, factory, codex, mux, opencode, junie, kiro, windsurf, cody, and kilocode. + | + | Examples: + | bd setup cursor # Install Cursor IDE integration (rules + agent hooks) + | bd setup cursor --global # Install global Cursor hooks (~/.cursor/hooks.json) + | bd setup kiro # Install Kiro steering guidance + | bd setup codex # Install Codex skill + AGENTS.md guidance + native hooks + | bd setup codex --global # Install global Codex skill + guidance + native hooks + | bd setup copilot # Install Copilot CLI plugin + repository instructions + | bd setup mux --project # Install Mux workspace layer (.mux/AGENTS.md) + | bd setup mux --global # Install Mux global layer (~/.mux/AGENTS.md) + | bd setup mux --project --global # Install both Mux layers + | bd setup --list # Show all available recipes + | bd setup --print # Print the template to stdout + | bd setup -o rules.md # Write template to custom path + | bd setup --add myeditor .myeditor/rules.md # Add custom recipe + | + | Use 'bd setup --check' to verify installation status. + | Use 'bd setup --remove' to uninstall. + flag --add type=string default="" usage="Add a custom recipe with given name" + flag --check type=bool default="false" usage="Check if integration is installed" + flag --global type=bool default="false" usage="Install globally (claude/codex/cursor/mux; writes to ~/.claude/settings.json, $CODEX_HOME/AGENTS.md or ~/.codex/AGENTS.md, ~/.cursor/hooks.json, or ~/.mux/AGENTS.md)" + flag --list type=bool default="false" usage="List all available recipes" + flag --output -o type=string default="" usage="Write template to custom path" + flag --print type=bool default="false" usage="Print the template to stdout" + flag --project type=bool default="false" usage="Install for this project only (gemini/mux)" + flag --remove type=bool default="false" usage="Remove the integration" + flag --stealth type=bool default="false" usage="Use stealth mode (claude/gemini)" + +bd ship + use: "ship " + short: "Publish a capability for cross-project dependencies" + hidden: false + runnable: true + long: + | Ship a capability to satisfy cross-project dependencies. + | + | This command: + | 1. Finds issue with export: label + | 2. Validates issue is closed (or --force to override) + | 3. Adds provides: label + | + | External projects can depend on this capability using: + | bd dep add external:: + | + | The capability is resolved when the external project has a closed issue + | with the provides: label. + | + | Examples: + | bd ship mol-run-assignee # Ship the mol-run-assignee capability + | bd ship mol-run-assignee --force # Ship even if issue is not closed + | bd ship mol-run-assignee --dry-run # Preview without making changes + flag --dry-run type=bool default="false" usage="Preview without making changes" + flag --force type=bool default="false" usage="Ship even if issue is not closed" + +bd show + use: "show [id...] [--id=...] [--current]" + aliases: ["view"] + short: "Show issue details" + group: "issues" + hidden: false + runnable: true + flag --as-of type=string default="" usage="Show issue as it existed at a specific commit hash or branch (requires Dolt; commit hashes must be the full 32-character hash from 'bd history', not a prefix)" + flag --brief-deps type=bool default="false" usage="Reduce each dependency to its identity fields in JSON output (--json only; drops description, design, notes and acceptance criteria)" + flag --children type=bool default="false" usage="Show only the children of this issue" + flag --comments-tail type=int default="0" usage="Render only the last N comments in text output, preceded by one elision line naming how many older ones were hidden (text output only; 0, the default, renders every comment unchanged; N must be >= 0)" + flag --current type=bool default="false" usage="Show the currently active issue (in-progress, hooked, or last touched)" + flag --id type=stringArray default="[]" usage="Issue ID (use for IDs that look like flags, e.g., --id=gt--xyz)" + flag --include-comments type=bool default="false" usage="Stream full comment bodies in JSON output (--json only; may be slow on issues with many comments). Without it the comments field is absent entirely, not truncated: an issue with comments carries comment_count and comments_omitted=true, so check those before reading an absent comments field as none." + flag --include-dependents type=bool default="false" usage="Stream full dependent issues in JSON output (--json only; may be slow on hub beads)" + flag --local-time type=bool default="false" usage="Show timestamps in local time instead of UTC (date-only fields are always local)" + flag --long type=bool default="false" usage="Show all available fields (extended metadata, agent identity, gate fields, etc.)" + flag --refs type=bool default="false" usage="Show issues that reference this issue (reverse lookup)" + flag --short type=bool default="false" usage="Show compact one-line output per issue" + flag --thread type=bool default="false" usage="Show full conversation thread (for messages)" + flag --watch -w type=bool default="false" usage="Watch for changes and auto-refresh display" + +bd sql + use: "sql " + short: "Execute raw SQL against the beads database" + group: "maint" + hidden: false + runnable: true + long: + | Execute a raw SQL query against the underlying database (Dolt). + | + | Useful for debugging, maintenance, and working around bugs in higher-level commands. + | + | Examples: + | bd sql 'SELECT COUNT(*) FROM issues' + | bd sql 'SELECT id, title FROM issues WHERE status = "open" LIMIT 5' + | bd sql 'DELETE FROM dirty_issues WHERE issue_id = "bd-abc123"' + | bd sql --csv 'SELECT id, title, status FROM issues' + | + | The query is passed directly to the database. Reads (SELECT, WITH ... SELECT, + | SHOW, EXPLAIN) return results as a table (or JSON/CSV with --json/--csv). Writes + | (INSERT, UPDATE, DELETE, DDL) report the number of rows affected. Statements + | that may both write and return rows, such as CALL, are committed and print any + | rows they return (or "OK"). + | + | In proxied-server mode, multiple statements separated by ';' run as a single + | committed batch and report "OK", and --database runs the query against a + | different server database (equivalent to a session USE) without changing the + | project's configured database. + | + | WARNING: Direct database access bypasses the storage layer. Use with caution. + flag --csv type=bool default="false" usage="Output results in CSV format" + +bd stale + use: "stale" + short: "Show stale issues (not updated recently)" + group: "views" + hidden: false + runnable: true + long: + | Show issues that haven't been updated recently and may need attention. + | This helps identify: + | - In-progress issues with no recent activity (may be abandoned) + | - Open issues that have been forgotten + | - Issues that might be outdated or no longer relevant + flag --days -d type=int default="30" usage="Issues not updated in this many days" + flag --exclude-label type=stringSlice default="[]" usage="Exclude issues that have ANY of these labels" + flag --label -l type=stringSlice default="[]" usage="Filter by labels (AND: must have ALL). Can combine with --label-any" + flag --label-any type=stringSlice default="[]" usage="Filter by labels (OR: must have AT LEAST ONE). Can combine with --label" + flag --limit -n type=int default="50" usage="Maximum issues to show" + flag --status -s type=string default="" usage="Filter by status (open|in_progress|blocked|deferred)" + +bd state + use: "state " + short: "Query the current value of a state dimension" + group: "issues" + hidden: false + runnable: true + long: + | Query the current value of a state dimension from an issue's labels. + | + | State labels follow the convention :, for example: + | patrol:active + | mode:degraded + | health:healthy + | + | This command extracts the value for a given dimension. + | + | Examples: + | bd state witness-abc patrol # Output: active + | bd state witness-abc mode # Output: normal + | bd state witness-abc health # Output: healthy + +bd state list + use: "list " + short: "List all state dimensions on an issue" + hidden: false + runnable: true + long: + | List all state labels (dimension:value format) on an issue. + | + | This filters labels to only show those following the state convention. + | + | Example: + | bd state list witness-abc + | # Output: + | # patrol: active + | # mode: normal + | # health: healthy + +bd status + use: "status" + aliases: ["stats"] + short: "Show issue database overview and statistics" + group: "views" + hidden: false + runnable: true + long: + | Show a quick snapshot of the issue database state and statistics. + | + | This command provides a summary of issue counts by state (open, in_progress, + | blocked, closed), ready work, extended statistics (pinned issues, + | average lead time), and recent activity over the last 24 hours from git history. + | + | Similar to how 'git status' shows working tree state, 'bd status' gives you + | a quick overview of your issue database without needing multiple queries. + | + | Use cases: + | - Quick project health check + | - Onboarding for new contributors + | - Integration with shell prompts or CI/CD + | - Daily standup reference + | - Fast CI status checks that don't need blocked-count accuracy + | + | Examples: + | bd status # Show summary with activity + | bd status --no-activity # Skip git activity (faster) + | bd status --no-blocked # Skip slow blocked-count scan (faster) + | bd stats --no-blocked --json # JSON output without blocked count + | bd status --json # JSON format output + | bd status --assigned # Show issues assigned to current user + | bd stats # Alias for bd status + flag --all type=bool default="false" usage="Show all issues (default behavior)" + flag --assigned type=bool default="false" usage="Show issues assigned to current user" + flag --no-activity type=bool default="false" usage="Skip git activity summary (faster)" + flag --no-blocked type=bool default="false" usage="Skip blocked-count computation (faster on large rigs; not supported in proxied-server mode)" + +bd statuses + use: "statuses" + short: "List valid issue statuses" + group: "views" + hidden: false + runnable: true + long: + | List all valid issue statuses and their categories. + | + | Built-in statuses (open, in_progress, blocked, etc.) are always valid. + | Additional statuses can be configured via status.custom: + | + | bd config set status.custom "in_review:active,qa_testing:wip,on_hold:frozen" + | + | Categories control behavior: + | active — appears in 'bd ready' and default 'bd list' + | wip — excluded from 'bd ready', visible in default 'bd list' + | done — excluded from 'bd ready' and default 'bd list' + | frozen — excluded from 'bd ready' and default 'bd list' + | + | Statuses without a category (legacy format) are valid but excluded from 'bd ready'. + | + | Examples: + | bd statuses # List all statuses with icons and categories + | bd statuses --json # Output as JSON + | + +bd supersede + use: "supersede --with " + short: "Mark an issue as superseded by a replacement issue" + group: "deps" + hidden: false + runnable: true + long: + | Mark an issue as superseded by a different (replacement) issue. + | + | The superseded issue is automatically closed with a reference to the replacement. + | Useful for design docs, specs, and evolving artifacts. + | + | Examples: + | bd supersede bd-old --with bd-new # Mark bd-old as superseded by bd-new + flag --with type=string default="" annotations={cobra_annotation_bash_completion_one_required_flag=["true"]} usage="Replacement issue ID (required)" + +bd swarm + use: "swarm" + short: "Swarm management for structured epics" + group: "deps" + hidden: false + runnable: false + long: + | Swarm management commands for coordinating parallel work on epics. + | + | A swarm is a structured body of work defined by an epic and its children, + | with dependencies forming a DAG (directed acyclic graph) of work. + +bd swarm create + use: "create [epic-id]" + short: "Create a swarm molecule from an epic" + hidden: false + runnable: true + long: + | Create a swarm molecule to orchestrate parallel work on an epic. + | + | The swarm molecule: + | - Links to the epic it orchestrates + | - Has mol_type=swarm for discovery + | - Specifies a coordinator (optional) + | - Can be picked up by any coordinator agent + | + | If given a single issue (not an epic), it will be auto-wrapped: + | - Creates an epic with that issue as its only child + | - Then creates the swarm molecule for that epic + | + | Examples: + | bd swarm create bd-epic-123 # Create swarm for epic + | bd swarm create bd-epic-123 --coordinator=observer/ # With specific coordinator + | bd swarm create bd-task-456 # Auto-wrap single issue + flag --coordinator type=string default="" usage="Coordinator address (e.g., my-project/witness)" + flag --force type=bool default="false" usage="Create new swarm even if one already exists" + +bd swarm list + use: "list" + short: "List all swarm molecules" + hidden: false + runnable: true + long: + | List all swarm molecules with their status. + | + | Shows each swarm molecule with: + | - Progress (completed/total issues) + | - Active workers + | - Epic ID and title + | + | Examples: + | bd swarm list # List all swarms + | bd swarm list --json # Machine-readable output + +bd swarm status + use: "status [epic-or-swarm-id]" + short: "Show current swarm status" + hidden: false + runnable: true + long: + | Show the current status of a swarm, computed from beads. + | + | Accepts either: + | - An epic ID (shows status for that epic's children) + | - A swarm molecule ID (follows the link to find the epic) + | + | Displays issues grouped by state: + | - Completed: Closed issues + | - Active: Issues currently in_progress (with assignee) + | - Ready: Open issues with all dependencies satisfied + | - Blocked: Open issues waiting on dependencies + | + | The status is COMPUTED from beads, not stored separately. + | If beads changes, status changes. + | + | Examples: + | bd swarm status gt-epic-123 # Show swarm status by epic + | bd swarm status gt-swarm-456 # Show status via swarm molecule + | bd swarm status gt-epic-123 --json # Machine-readable output + +bd swarm validate + use: "validate [epic-id]" + short: "Validate epic structure for swarming" + hidden: false + runnable: true + long: + | Validate an epic's structure to ensure it's ready for swarm execution. + | + | Checks for: + | - Correct dependency direction (requirement-based, not temporal) + | - Orphaned issues (roots with no dependents) + | - Missing dependencies (leaves that should depend on something) + | - Cycles (impossible to resolve) + | - Disconnected subgraphs + | + | Reports: + | - Ready fronts (waves of parallel work) + | - Estimated worker-sessions + | - Maximum parallelism + | - Warnings for potential issues + | + | Examples: + | bd swarm validate gt-epic-123 # Validate epic structure + | bd swarm validate gt-epic-123 --verbose # Include detailed issue graph + flag --verbose type=bool default="false" usage="Include detailed issue graph in output" + +bd sync + use: "sync" + short: "Pull, check for conflicts, repair is_blocked, and push (the federation loop)" + group: "sync" + hidden: false + runnable: true + long: + | Run one full synchronization cycle against the Dolt remote. + | + | This is the loop every multi-machine beads deployment otherwise hand-rolls in + | shell: + | + | 1. pull from the remote + | 2. check for merge conflicts POSITIVELY, from the merge's own conflict rows + | and from Dolt's conflict tables — never inferred from the pull's exit + | status, which is not a trustworthy conflict signal in either direction + | 3. recompute the denormalized is_blocked flag, so dependency edges merged in + | from another replica do not leave 'bd ready' stale + | 4. push, retrying a bounded number of times when another replica wins the + | push race + | + | The repair in step 3 refuses to run while another writer has uncommitted changes + | to issues/dependencies. That is transient and not this sync's doing, so it is + | retried on the same budget as a push race rather than failing the run. A working + | set that is NOT transient exits 4 instead, because no amount of retrying will + | ever publish and only an operator can clear it. Two kinds of evidence say so: + | constraint violations on the dirty tables are detected positively and escalate + | on the very attempt that finds them; an abandoned uncommitted edit has no such + | positive signal, so it is only inferred once the same pending graph edits have + | blocked every attempt of several consecutive runs. + | + | Conflicts sync cannot resolve safely are NEVER auto-resolved: it halts before + | recomputing or pushing and exits 2, and repeated runs keep halting the same way + | until an operator resolves the divergence. (The pull underneath does auto-settle + | the conflict classes it can settle convergently — machine-local metadata, + | audit-only dependency rows, and last-write-wins on issue cells. Anything beyond + | those halts here.) Whether the halted merge was aborted or left live in the + | working set depends on the pull route, so the halt message reports which. + | + | Exit codes (a sync timer can branch on these without parsing output): + | + | 0 synced, or nothing to do + | 1 error (transport, auth, storage) + | 2 merge conflict — halted, nothing pushed, resolve it by hand + | 3 retries exhausted (push race, or a concurrent writer's dirty working set) + | — transient, nothing pushed, retry on the next tick + | 4 the dirty working set is stuck, not busy: identical pending graph edits + | blocked every attempt of several consecutive runs — nothing pushed, and no + | later tick will publish until an operator clears it + | + | On the default-remote path, a rig with no Dolt remote yet but a git origin + | configured adopts that origin as its Dolt remote first, exactly as 'bd dolt push' + | does — so 'bd sync' works as a first-time federation bring-up step instead of + | reporting 'no remote' and doing nothing. Passing --remote never adopts anything. + | + | This is not 'bd federation sync', which syncs with named peer towns and takes a + | --strategy ours|theirs to resolve whatever conflicts it meets. 'bd sync' targets + | the configured remote and has no such switch: what it cannot settle, it halts on. + | + | Examples: + | bd sync # sync with the default remote + | bd sync --remote mini # sync with a specific remote + | bd sync --attempts 5 # allow more push-race retries + | bd sync --json # machine-parseable outcome + flag --attempts type=int default="3" usage="Maximum pull/push attempts before reporting a transient retry exhaustion (exit 3)" + flag --no-adopt type=bool default="false" usage="Never derive a Dolt remote from git origin (also BD_NO_REMOTE_ADOPT=1)" + flag --remote type=string default="" usage="Sync with a specific named remote instead of the default" + flag --yes -y type=bool default="false" usage="Consent to adopting a Dolt remote derived from git origin when none is configured" + +bd tag + use: "tag